From 0fbf8b172a084a05f2da74f0c252b60f33386d13 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 13 Aug 2019 09:32:59 +1000 Subject: [PATCH] Websocket orderbook buffering (#333) * Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process. * Moves namespaces again * Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests * Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps * Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests * Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic * Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation * Implements orderbook update for okgroup, but forgets about the orderbook hash checking * Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests. * Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains * Fixes Gemini rate limit and parsing. Removes comments and fixes typos * Fixes bitfinex orderbook processing * Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage * Adds surprise LakeBTC websocket support * Fixes data race * Fixes rebasing issues due to namespace movements * Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments * Adds poloniex orderook sorting support * Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting * Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well * Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines * Fixes test with exchange name output change * Fixes issues where copy and paste and replace all were used poorly --- config/config.go | 7 +- config/config_test.go | 5 +- exchanges/alphapoint/alphapoint_wrapper.go | 2 +- exchanges/anx/anx.go | 2 +- exchanges/anx/anx_wrapper.go | 2 +- exchanges/binance/binance.go | 11 +- exchanges/binance/binance_websocket.go | 177 +++--- exchanges/binance/binance_wrapper.go | 2 +- exchanges/bitfinex/bitfinex.go | 10 +- exchanges/bitfinex/bitfinex_test.go | 2 +- exchanges/bitfinex/bitfinex_websocket.go | 181 ++---- exchanges/bitfinex/bitfinex_wrapper.go | 2 +- exchanges/bitflyer/bitflyer.go | 2 +- exchanges/bitflyer/bitflyer_wrapper.go | 2 +- exchanges/bithumb/bithumb.go | 2 +- exchanges/bithumb/bithumb_wrapper.go | 2 +- exchanges/bitmex/bitmex.go | 10 +- exchanges/bitmex/bitmex_test.go | 2 +- exchanges/bitmex/bitmex_websocket.go | 149 ++--- exchanges/bitmex/bitmex_wrapper.go | 2 +- exchanges/bitstamp/bitstamp.go | 10 +- exchanges/bitstamp/bitstamp_websocket.go | 48 +- exchanges/bitstamp/bitstamp_wrapper.go | 2 +- exchanges/bittrex/bittrex.go | 2 +- exchanges/bittrex/bittrex_wrapper.go | 2 +- exchanges/btcmarkets/btcmarkets.go | 2 +- exchanges/btcmarkets/btcmarkets_wrapper.go | 2 +- exchanges/btse/btse.go | 10 +- exchanges/btse/btse_websocket.go | 22 +- exchanges/btse/btse_wrapper.go | 2 +- exchanges/coinbasepro/coinbasepro.go | 10 +- exchanges/coinbasepro/coinbasepro_test.go | 2 +- .../coinbasepro/coinbasepro_websocket.go | 56 +- exchanges/coinbasepro/coinbasepro_wrapper.go | 2 +- exchanges/coinut/coinut.go | 13 +- exchanges/coinut/coinut_test.go | 2 +- exchanges/coinut/coinut_websocket.go | 60 +- exchanges/coinut/coinut_wrapper.go | 2 +- exchanges/exchange.go | 5 +- exchanges/exchange_test.go | 7 +- exchanges/exmo/exmo.go | 2 +- exchanges/exmo/exmo_wrapper.go | 2 +- exchanges/gateio/gateio.go | 10 +- exchanges/gateio/gateio_test.go | 2 +- exchanges/gateio/gateio_websocket.go | 52 +- exchanges/gateio/gateio_wrapper.go | 2 +- exchanges/gemini/gemini.go | 14 +- exchanges/gemini/gemini_test.go | 2 +- exchanges/gemini/gemini_types.go | 2 +- exchanges/gemini/gemini_websocket.go | 90 ++- exchanges/gemini/gemini_wrapper.go | 2 +- exchanges/hitbtc/hitbtc.go | 10 +- exchanges/hitbtc/hitbtc_test.go | 2 +- exchanges/hitbtc/hitbtc_websocket.go | 44 +- exchanges/hitbtc/hitbtc_wrapper.go | 2 +- exchanges/huobi/huobi.go | 10 +- exchanges/huobi/huobi_test.go | 2 +- exchanges/huobi/huobi_websocket.go | 30 +- exchanges/huobi/huobi_wrapper.go | 2 +- exchanges/huobihadax/huobihadax.go | 10 +- exchanges/huobihadax/huobihadax_test.go | 2 +- exchanges/huobihadax/huobihadax_websocket.go | 30 +- exchanges/huobihadax/huobihadax_wrapper.go | 2 +- exchanges/itbit/itbit.go | 2 +- exchanges/itbit/itbit_wrapper.go | 2 +- exchanges/kraken/kraken.go | 11 +- exchanges/kraken/kraken_test.go | 106 +--- exchanges/kraken/kraken_websocket.go | 259 +------- exchanges/kraken/kraken_wrapper.go | 2 +- exchanges/lakebtc/lakebtc.go | 26 +- exchanges/lakebtc/lakebtc_test.go | 94 ++- exchanges/lakebtc/lakebtc_types.go | 29 + exchanges/lakebtc/lakebtc_websocket.go | 248 ++++++++ exchanges/lakebtc/lakebtc_wrapper.go | 5 +- exchanges/localbitcoins/localbitcoins.go | 2 +- .../localbitcoins/localbitcoins_wrapper.go | 2 +- exchanges/okcoin/okcoin.go | 3 +- exchanges/okcoin/okcoin_test.go | 5 +- exchanges/okex/okex.go | 3 +- exchanges/okex/okex_test.go | 11 +- exchanges/okgroup/okgroup.go | 9 +- exchanges/okgroup/okgroup_websocket.go | 73 +-- exchanges/okgroup/okgroup_wrapper.go | 2 +- exchanges/poloniex/poloniex.go | 10 +- exchanges/poloniex/poloniex_test.go | 2 +- exchanges/poloniex/poloniex_websocket.go | 47 +- exchanges/poloniex/poloniex_wrapper.go | 2 +- .../sharedtestvalues/sharedtestvalues.go | 2 +- .../wshandler/wshandler.go} | 406 ++++++------ .../wshandler/wshandler_test.go} | 203 +----- .../wshandler/wshandler_types.go} | 30 +- .../websocket/wsorderbook/wsorderbook.go | 242 ++++++++ .../websocket/wsorderbook/wsorderbook_test.go | 582 ++++++++++++++++++ .../wsorderbook/wsorderbook_types.go | 34 + exchanges/wshandler/websocket_connection.go | 177 ------ .../wshandler/websocket_connection_test.go | 203 ------ .../wshandler/websocket_connection_types.go | 25 - exchanges/yobit/yobit.go | 2 +- exchanges/yobit/yobit_wrapper.go | 2 +- exchanges/zb/zb.go | 10 +- exchanges/zb/zb_test.go | 2 +- exchanges/zb/zb_websocket.go | 19 +- exchanges/zb/zb_wrapper.go | 2 +- go.mod | 1 + go.sum | 2 + routines.go | 2 +- testdata/configtest.json | 33 +- tools/exchange_template/exchange_template.go | 3 +- tools/exchange_template/main_file.tmpl | 2 +- tools/exchange_template/wrapper_file.tmpl | 6 +- 110 files changed, 2197 insertions(+), 1909 deletions(-) create mode 100644 exchanges/lakebtc/lakebtc_websocket.go rename exchanges/{wshandler/websocket.go => websocket/wshandler/wshandler.go} (74%) rename exchanges/{wshandler/websocket_test.go => websocket/wshandler/wshandler_test.go} (62%) rename exchanges/{wshandler/websocket_types.go => websocket/wshandler/wshandler_types.go} (90%) create mode 100644 exchanges/websocket/wsorderbook/wsorderbook.go create mode 100644 exchanges/websocket/wsorderbook/wsorderbook_test.go create mode 100644 exchanges/websocket/wsorderbook/wsorderbook_types.go delete mode 100644 exchanges/wshandler/websocket_connection.go delete mode 100644 exchanges/wshandler/websocket_connection_test.go delete mode 100644 exchanges/wshandler/websocket_connection_types.go diff --git a/config/config.go b/config/config.go index 125a2b1d..a3170688 100644 --- a/config/config.go +++ b/config/config.go @@ -37,6 +37,7 @@ const ( configDefaultHTTPTimeout = time.Second * 15 configDefaultWebsocketResponseCheckTimeout = time.Millisecond * 30 configDefaultWebsocketResponseMaxLimit = time.Second * 7 + configDefaultWebsocketOrderbookBufferLimit = 5 configMaxAuthFailres = 3 defaultNTPAllowedDifference = 50000000 defaultNTPAllowedNegativeDifference = 50000000 @@ -161,6 +162,7 @@ type ExchangeConfig struct { HTTPTimeout time.Duration `json:"httpTimeout"` WebsocketResponseCheckTimeout time.Duration `json:"websocketResponseCheckTimeout"` WebsocketResponseMaxLimit time.Duration `json:"websocketResponseMaxLimit"` + WebsocketOrderbookBufferLimit int `json:"websocketOrderbookBufferLimit"` HTTPUserAgent string `json:"httpUserAgent"` HTTPDebugging bool `json:"httpDebugging"` AuthenticatedAPISupport bool `json:"authenticatedApiSupport"` @@ -837,7 +839,10 @@ func (c *Config) CheckExchangeConfigValues() error { log.Warnf("Exchange %s Websocket response max limit value not set, defaulting to %v.", c.Exchanges[i].Name, configDefaultWebsocketResponseMaxLimit) c.Exchanges[i].WebsocketResponseMaxLimit = configDefaultWebsocketResponseMaxLimit } - + if c.Exchanges[i].WebsocketOrderbookBufferLimit <= 0 { + log.Warnf("Exchange %s Websocket orderbook buffer limit value not set, defaulting to %v.", c.Exchanges[i].Name, configDefaultWebsocketOrderbookBufferLimit) + c.Exchanges[i].WebsocketOrderbookBufferLimit = configDefaultWebsocketOrderbookBufferLimit + } err := c.CheckPairConsistency(c.Exchanges[i].Name) if err != nil { log.Errorf("Exchange %s: CheckPairConsistency error: %s", c.Exchanges[i].Name, err) diff --git a/config/config_test.go b/config/config_test.go index f979dcbb..eed76da8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -666,6 +666,7 @@ func TestCheckExchangeConfigValues(t *testing.T) { checkExchangeConfigValues.Exchanges[0].WebsocketResponseMaxLimit = 0 checkExchangeConfigValues.Exchanges[0].WebsocketResponseCheckTimeout = 0 + checkExchangeConfigValues.Exchanges[0].WebsocketOrderbookBufferLimit = 0 checkExchangeConfigValues.Exchanges[0].HTTPTimeout = 0 err = checkExchangeConfigValues.CheckExchangeConfigValues() if err != nil { @@ -682,8 +683,8 @@ func TestCheckExchangeConfigValues(t *testing.T) { t.Fatalf("Test failed. Expected exchange %s to have updated WebsocketResponseMaxLimit value", checkExchangeConfigValues.Exchanges[0].Name) } - if checkExchangeConfigValues.Exchanges[0].WebsocketResponseCheckTimeout == 0 { - t.Fatalf("Test failed. Expected exchange %s to have updated WebsocketResponseCheckTimeout value", checkExchangeConfigValues.Exchanges[0].Name) + if checkExchangeConfigValues.Exchanges[0].WebsocketOrderbookBufferLimit == 0 { + t.Fatalf("Test failed. Expected exchange %s to have updated WebsocketOrderbookBufferLimit value", checkExchangeConfigValues.Exchanges[0].Name) } checkExchangeConfigValues.Exchanges[0].APIKey = "Key" diff --git a/exchanges/alphapoint/alphapoint_wrapper.go b/exchanges/alphapoint/alphapoint_wrapper.go index 72c959e4..80e106f8 100644 --- a/exchanges/alphapoint/alphapoint_wrapper.go +++ b/exchanges/alphapoint/alphapoint_wrapper.go @@ -11,7 +11,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // GetAccountInfo retrieves balances for all enabled currencies on the diff --git a/exchanges/anx/anx.go b/exchanges/anx/anx.go index 13f8015d..25e7992f 100644 --- a/exchanges/anx/anx.go +++ b/exchanges/anx/anx.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/anx/anx_wrapper.go b/exchanges/anx/anx_wrapper.go index ac5bac0a..fecea716 100644 --- a/exchanges/anx/anx_wrapper.go +++ b/exchanges/anx/anx_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/binance/binance.go b/exchanges/binance/binance.go index ca81161a..da164add 100644 --- a/exchanges/binance/binance.go +++ b/exchanges/binance/binance.go @@ -16,7 +16,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -101,6 +101,7 @@ func (b *Binance) SetDefaults() { wshandler.WebsocketOrderbookSupported b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + b.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup takes in the supplied exchange configuration details and sets params @@ -152,7 +153,6 @@ func (b *Binance) Setup(exch *config.ExchangeConfig) { if err != nil { log.Fatal(err) } - b.WebsocketConn = &wshandler.WebsocketConnection{ ExchangeName: b.Name, URL: b.Websocket.GetWebsocketURL(), @@ -161,6 +161,13 @@ func (b *Binance) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + b.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + true, + true, + false, + exch.Name) } } diff --git a/exchanges/binance/binance_websocket.go b/exchanges/binance/binance_websocket.go index db1c1642..71b5e78a 100644 --- a/exchanges/binance/binance_websocket.go +++ b/exchanges/binance/binance_websocket.go @@ -6,7 +6,6 @@ import ( "net/http" "strconv" "strings" - "sync" "time" "github.com/gorilla/websocket" @@ -15,112 +14,14 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" ) const ( binanceDefaultWebsocketURL = "wss://stream.binance.com:9443" ) -var lastUpdateID map[string]int64 -var m sync.Mutex - -// SeedLocalCache seeds depth data -func (b *Binance) SeedLocalCache(p currency.Pair) error { - var newOrderBook orderbook.Base - - formattedPair := exchange.FormatExchangeCurrency(b.Name, p) - - orderbookNew, err := b.GetOrderBook( - OrderBookDataRequestParams{ - Symbol: formattedPair.String(), - Limit: 1000, - }) - - if err != nil { - return err - } - - m.Lock() - if lastUpdateID == nil { - lastUpdateID = make(map[string]int64) - } - - lastUpdateID[formattedPair.String()] = orderbookNew.LastUpdateID - m.Unlock() - - for _, bids := range orderbookNew.Bids { - newOrderBook.Bids = append(newOrderBook.Bids, - orderbook.Item{Amount: bids.Quantity, Price: bids.Price}) - } - for _, Asks := range orderbookNew.Asks { - newOrderBook.Asks = append(newOrderBook.Asks, - orderbook.Item{Amount: Asks.Quantity, Price: Asks.Price}) - } - - newOrderBook.Pair = currency.NewPairFromString(formattedPair.String()) - newOrderBook.AssetType = ticker.Spot - - return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, b.GetName(), false) -} - -// UpdateLocalCache updates and returns the most recent iteration of the orderbook -func (b *Binance) UpdateLocalCache(ob *WebsocketDepthStream) error { - m.Lock() - ID, ok := lastUpdateID[ob.Pair] - if !ok { - m.Unlock() - return fmt.Errorf("%v - Unable to find lastUpdateID", b.Name) - } - - if ob.LastUpdateID+1 <= ID || ID >= ob.LastUpdateID+1 { - // Drop update, out of order - m.Unlock() - return nil - } - - lastUpdateID[ob.Pair] = ob.LastUpdateID - m.Unlock() - - var updateBid, updateAsk []orderbook.Item - - for _, bidsToUpdate := range ob.UpdateBids { - var priceToBeUpdated orderbook.Item - for i, bids := range bidsToUpdate.([]interface{}) { - switch i { - case 0: - priceToBeUpdated.Price, _ = strconv.ParseFloat(bids.(string), 64) - case 1: - priceToBeUpdated.Amount, _ = strconv.ParseFloat(bids.(string), 64) - } - } - updateBid = append(updateBid, priceToBeUpdated) - } - - for _, asksToUpdate := range ob.UpdateAsks { - var priceToBeUpdated orderbook.Item - for i, asks := range asksToUpdate.([]interface{}) { - switch i { - case 0: - priceToBeUpdated.Price, _ = strconv.ParseFloat(asks.(string), 64) - case 1: - priceToBeUpdated.Amount, _ = strconv.ParseFloat(asks.(string), 64) - } - } - updateAsk = append(updateAsk, priceToBeUpdated) - } - - updatedTime := time.Unix(ob.Timestamp, 0) - currencyPair := currency.NewPairFromString(ob.Pair) - - return b.Websocket.Orderbook.Update(updateBid, - updateAsk, - currencyPair, - updatedTime, - b.GetName(), - "SPOT") -} - // WSConnect intiates a websocket connection func (b *Binance) WSConnect() error { if !b.Websocket.IsEnabled() || !b.IsEnabled() { @@ -175,11 +76,9 @@ func (b *Binance) WSConnect() error { // WsHandleData handles websocket data from WsReadData func (b *Binance) WsHandleData() { b.Websocket.Wg.Add(1) - defer func() { b.Websocket.Wg.Done() }() - for { select { case <-b.Websocket.ShutdownC: @@ -234,7 +133,7 @@ func (b *Binance) WsHandleData() { Price: price, Amount: amount, Exchange: b.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, Side: trade.EventType, } continue @@ -309,7 +208,7 @@ func (b *Binance) WsHandleData() { currencyPair := currency.NewPairFromString(depth.Pair) b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Pair: currencyPair, - Asset: "SPOT", + Asset: orderbook.Spot, Exchange: b.GetName(), } continue @@ -317,3 +216,71 @@ func (b *Binance) WsHandleData() { } } } + +// SeedLocalCache seeds depth data +func (b *Binance) SeedLocalCache(p currency.Pair) error { + var newOrderBook orderbook.Base + formattedPair := exchange.FormatExchangeCurrency(b.Name, p) + orderbookNew, err := b.GetOrderBook( + OrderBookDataRequestParams{ + Symbol: formattedPair.String(), + Limit: 1000, + }) + if err != nil { + return err + } + + for i := range orderbookNew.Bids { + newOrderBook.Bids = append(newOrderBook.Bids, + orderbook.Item{Amount: orderbookNew.Bids[i].Quantity, Price: orderbookNew.Bids[i].Price}) + } + for i := range orderbookNew.Asks { + newOrderBook.Asks = append(newOrderBook.Asks, + orderbook.Item{Amount: orderbookNew.Asks[i].Quantity, Price: orderbookNew.Asks[i].Price}) + } + + newOrderBook.LastUpdated = time.Unix(orderbookNew.LastUpdateID, 0) + newOrderBook.Pair = currency.NewPairFromString(formattedPair.String()) + newOrderBook.AssetType = ticker.Spot + + return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false) +} + +// UpdateLocalCache updates and returns the most recent iteration of the orderbook +func (b *Binance) UpdateLocalCache(wsdp *WebsocketDepthStream) error { + var updateBid, updateAsk []orderbook.Item + for i := range wsdp.UpdateBids { + var priceToBeUpdated orderbook.Item + for i, bids := range wsdp.UpdateBids[i].([]interface{}) { + switch i { + case 0: + priceToBeUpdated.Price, _ = strconv.ParseFloat(bids.(string), 64) + case 1: + priceToBeUpdated.Amount, _ = strconv.ParseFloat(bids.(string), 64) + } + } + updateBid = append(updateBid, priceToBeUpdated) + } + + for i := range wsdp.UpdateAsks { + var priceToBeUpdated orderbook.Item + for i, asks := range wsdp.UpdateAsks[i].([]interface{}) { + switch i { + case 0: + priceToBeUpdated.Price, _ = strconv.ParseFloat(asks.(string), 64) + case 1: + priceToBeUpdated.Amount, _ = strconv.ParseFloat(asks.(string), 64) + } + } + updateAsk = append(updateAsk, priceToBeUpdated) + } + currencyPair := currency.NewPairFromString(wsdp.Pair) + + return b.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{ + Bids: updateBid, + Asks: updateAsk, + CurrencyPair: currencyPair, + UpdateID: wsdp.LastUpdateID, + AssetType: orderbook.Spot, + }) +} diff --git a/exchanges/binance/binance_wrapper.go b/exchanges/binance/binance_wrapper.go index a14e7412..f0241904 100644 --- a/exchanges/binance/binance_wrapper.go +++ b/exchanges/binance/binance_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bitfinex/bitfinex.go b/exchanges/bitfinex/bitfinex.go index 8c2132e5..58fa3dc4 100644 --- a/exchanges/bitfinex/bitfinex.go +++ b/exchanges/bitfinex/bitfinex.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -120,6 +120,7 @@ func (b *Bitfinex) SetDefaults() { wshandler.WebsocketAuthenticatedEndpointsSupported b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + b.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup takes in the supplied exchange configuration details and sets params @@ -182,6 +183,13 @@ func (b *Bitfinex) Setup(exch *config.ExchangeConfig) { } b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + b.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + false, + false, + false, + exch.Name) } } diff --git a/exchanges/bitfinex/bitfinex_test.go b/exchanges/bitfinex/bitfinex_test.go index 5f3a1987..292c79b5 100644 --- a/exchanges/bitfinex/bitfinex_test.go +++ b/exchanges/bitfinex/bitfinex_test.go @@ -13,7 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please supply your own keys here to do better tests diff --git a/exchanges/bitfinex/bitfinex_websocket.go b/exchanges/bitfinex/bitfinex_websocket.go index 3f916a46..1e173d1f 100644 --- a/exchanges/bitfinex/bitfinex_websocket.go +++ b/exchanges/bitfinex/bitfinex_websocket.go @@ -13,7 +13,8 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -182,7 +183,11 @@ func (b *Bitfinex) WsDataHandler() { if stream.Type == websocket.TextMessage { var result interface{} - common.JSONDecode(stream.Raw, &result) + err = common.JSONDecode(stream.Raw, &result) + if err != nil { + b.Websocket.DataHandler <- err + return + } switch reflect.TypeOf(result).String() { case "map[string]interface {}": eventData := result.(map[string]interface{}) @@ -229,45 +234,40 @@ func (b *Bitfinex) WsDataHandler() { switch chanInfo.Channel { case "book": var newOrderbook []WebsocketBook + curr := currency.NewPairFromString(chanInfo.Pair) switch len(chanData) { case 2: data := chanData[1].([]interface{}) - for _, x := range data { - y := x.([]interface{}) + for i := range data { + y := data[i].([]interface{}) newOrderbook = append(newOrderbook, WebsocketBook{ Price: y[0].(float64), Count: int(y[1].(float64)), Amount: y[2].(float64)}) } - case 4: - newOrderbook = append(newOrderbook, WebsocketBook{ - Price: chanData[1].(float64), - Count: int(chanData[2].(float64)), - Amount: chanData[3].(float64)}) - } - - if len(newOrderbook) > 1 { - err := b.WsInsertSnapshot(currency.NewPairFromString(chanInfo.Pair), - "SPOT", + err := b.WsInsertSnapshot(curr, + orderbook.Spot, newOrderbook) if err != nil { b.Websocket.DataHandler <- fmt.Errorf("bitfinex_websocket.go inserting snapshot error: %s", err) } - continue + case 4: + newOrderbook = append(newOrderbook, WebsocketBook{ + Price: chanData[1].(float64), + Count: int(chanData[2].(float64)), + Amount: chanData[3].(float64)}) + err := b.WsUpdateOrderbook(curr, + orderbook.Spot, + newOrderbook) + + if err != nil { + b.Websocket.DataHandler <- fmt.Errorf("bitfinex_websocket.go updating orderbook error: %s", + err) + } } - - err := b.WsUpdateOrderbook(currency.NewPairFromString(chanInfo.Pair), - "SPOT", - newOrderbook[0]) - - if err != nil { - b.Websocket.DataHandler <- fmt.Errorf("bitfinex_websocket.go updating orderbook error: %s", - err) - } - case "ticker": b.Websocket.DataHandler <- wshandler.TickerData{ Quantity: chanData[8].(float64), @@ -276,7 +276,7 @@ func (b *Bitfinex) WsDataHandler() { LowPrice: chanData[10].(float64), Pair: currency.NewPairFromString(chanInfo.Pair), Exchange: b.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, } case "account": @@ -284,8 +284,8 @@ func (b *Bitfinex) WsDataHandler() { case bitfinexWebsocketPositionSnapshot: var positionSnapshot []WebsocketPosition data := chanData[2].([]interface{}) - for _, x := range data { - y := x.([]interface{}) + for i := range data { + y := data[i].([]interface{}) positionSnapshot = append(positionSnapshot, WebsocketPosition{ Pair: y[0].(string), @@ -317,8 +317,8 @@ func (b *Bitfinex) WsDataHandler() { case bitfinexWebsocketWalletSnapshot: data := chanData[2].([]interface{}) var walletSnapshot []WebsocketWallet - for _, x := range data { - y := x.([]interface{}) + for i := range data { + y := data[i].([]interface{}) walletSnapshot = append(walletSnapshot, WebsocketWallet{ Name: y[0].(string), @@ -342,8 +342,8 @@ func (b *Bitfinex) WsDataHandler() { case bitfinexWebsocketOrderSnapshot: var orderSnapshot []WebsocketOrder data := chanData[2].([]interface{}) - for _, x := range data { - y := x.([]interface{}) + for i := range data { + y := data[i].([]interface{}) orderSnapshot = append(orderSnapshot, WebsocketOrder{ OrderID: int64(y[0].(float64)), @@ -406,8 +406,8 @@ func (b *Bitfinex) WsDataHandler() { switch len(chanData) { case 2: data := chanData[1].([]interface{}) - for _, x := range data { - y := x.([]interface{}) + for i := range data { + y := data[i].([]interface{}) if _, ok := y[0].(string); ok { continue } @@ -445,7 +445,7 @@ func (b *Bitfinex) WsDataHandler() { Price: trades[0].Price, Amount: newAmount, Exchange: b.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, Side: side, } } @@ -462,31 +462,26 @@ func (b *Bitfinex) WsInsertSnapshot(p currency.Pair, assetType string, books []W if len(books) == 0 { return errors.New("bitfinex.go error - no orderbooks submitted") } - var bid, ask []orderbook.Item - for _, book := range books { - if book.Amount >= 0 { - bid = append(bid, orderbook.Item{Amount: book.Amount, Price: book.Price}) + for i := range books { + if books[i].Amount >= 0 { + bid = append(bid, orderbook.Item{Amount: books[i].Amount, Price: books[i].Price}) } else { - ask = append(ask, orderbook.Item{Amount: book.Amount * -1, Price: book.Price}) + ask = append(ask, orderbook.Item{Amount: books[i].Amount * -1, Price: books[i].Price}) } } - if len(bid) == 0 && len(ask) == 0 { return errors.New("bitfinex.go error - no orderbooks in item lists") } - var newOrderBook orderbook.Base newOrderBook.Asks = ask newOrderBook.AssetType = assetType newOrderBook.Bids = bid newOrderBook.Pair = p - - err := b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, b.GetName(), false) + err := b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false) if err != nil { return fmt.Errorf("bitfinex.go error - %s", err) } - b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p, Asset: assetType, Exchange: b.GetName()} @@ -495,80 +490,36 @@ func (b *Bitfinex) WsInsertSnapshot(p currency.Pair, assetType string, books []W // WsUpdateOrderbook updates the orderbook list, removing and adding to the // orderbook sides -func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType string, book WebsocketBook) error { +func (b *Bitfinex) WsUpdateOrderbook(p currency.Pair, assetType string, book []WebsocketBook) error { + orderbookUpdate := wsorderbook.WebsocketOrderbookUpdate{ + Asks: []orderbook.Item{}, + Bids: []orderbook.Item{}, + AssetType: assetType, + CurrencyPair: p, + } - if book.Count > 0 { - if book.Amount > 0 { - // Update/add bid - newBidPrice := orderbook.Item{Price: book.Price, Amount: book.Amount} - err := b.Websocket.Orderbook.Update([]orderbook.Item{newBidPrice}, - nil, - p, - time.Now(), - b.GetName(), - assetType) - - if err != nil { - return err + for i := 0; i < len(book); i++ { + switch { + case book[i].Count > 0: + if book[i].Amount > 0 { + // update bid + orderbookUpdate.Bids = append(orderbookUpdate.Bids, orderbook.Item{Amount: book[i].Amount, Price: book[i].Price}) + } else if book[i].Amount < 0 { + // update ask + orderbookUpdate.Asks = append(orderbookUpdate.Asks, orderbook.Item{Amount: book[i].Amount * -1, Price: book[i].Price}) + } + case book[i].Count == 0: + if book[i].Amount == 1 { + // delete bid + orderbookUpdate.Bids = append(orderbookUpdate.Bids, orderbook.Item{Amount: 0, Price: book[i].Price}) + } else if book[i].Amount == -1 { + // delete ask + orderbookUpdate.Asks = append(orderbookUpdate.Asks, orderbook.Item{Amount: 0, Price: book[i].Price}) } - - b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p, - Asset: assetType, - Exchange: b.GetName()} - - return nil } - - // Update/add ask - newAskPrice := orderbook.Item{Price: book.Price, Amount: book.Amount * -1} - err := b.Websocket.Orderbook.Update(nil, - []orderbook.Item{newAskPrice}, - p, - time.Now(), - b.GetName(), - assetType) - - if err != nil { - return err - } - - b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p, - Asset: assetType, - Exchange: b.GetName()} - - return nil } - - if book.Amount == 1 { - // Remove bid - bidPriceRemove := orderbook.Item{Price: book.Price, Amount: 0} - err := b.Websocket.Orderbook.Update([]orderbook.Item{bidPriceRemove}, - nil, - p, - time.Now(), - b.GetName(), - assetType) - - if err != nil { - return err - } - - b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: p, - Asset: assetType, - Exchange: b.GetName()} - - return nil - } - - // Remove from ask - askPriceRemove := orderbook.Item{Price: book.Price, Amount: 0} - err := b.Websocket.Orderbook.Update(nil, - []orderbook.Item{askPriceRemove}, - p, - time.Now(), - b.GetName(), - assetType) - + orderbookUpdate.UpdateTime = time.Now() + err := b.Websocket.Orderbook.Update(&orderbookUpdate) if err != nil { return err } diff --git a/exchanges/bitfinex/bitfinex_wrapper.go b/exchanges/bitfinex/bitfinex_wrapper.go index df814926..a5aadc1f 100644 --- a/exchanges/bitfinex/bitfinex_wrapper.go +++ b/exchanges/bitfinex/bitfinex_wrapper.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bitflyer/bitflyer.go b/exchanges/bitflyer/bitflyer.go index f9c61652..78dd3dc5 100644 --- a/exchanges/bitflyer/bitflyer.go +++ b/exchanges/bitflyer/bitflyer.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bitflyer/bitflyer_wrapper.go b/exchanges/bitflyer/bitflyer_wrapper.go index 97237b34..6f1053c0 100644 --- a/exchanges/bitflyer/bitflyer_wrapper.go +++ b/exchanges/bitflyer/bitflyer_wrapper.go @@ -9,7 +9,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bithumb/bithumb.go b/exchanges/bithumb/bithumb.go index 92338e46..31dd7c24 100644 --- a/exchanges/bithumb/bithumb.go +++ b/exchanges/bithumb/bithumb.go @@ -17,7 +17,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bithumb/bithumb_wrapper.go b/exchanges/bithumb/bithumb_wrapper.go index b3d2fff0..3d4cfe49 100644 --- a/exchanges/bithumb/bithumb_wrapper.go +++ b/exchanges/bithumb/bithumb_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bitmex/bitmex.go b/exchanges/bitmex/bitmex.go index 29238192..b1410eb5 100644 --- a/exchanges/bitmex/bitmex.go +++ b/exchanges/bitmex/bitmex.go @@ -15,7 +15,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -143,6 +143,7 @@ func (b *Bitmex) SetDefaults() { wshandler.WebsocketDeadMansSwitchSupported b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + b.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup takes in the supplied exchange configuration details and sets params @@ -201,6 +202,13 @@ func (b *Bitmex) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + b.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + false, + false, + true, + exch.Name) } } diff --git a/exchanges/bitmex/bitmex_test.go b/exchanges/bitmex/bitmex_test.go index 97fd7adf..cd519573 100644 --- a/exchanges/bitmex/bitmex_test.go +++ b/exchanges/bitmex/bitmex_test.go @@ -12,7 +12,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please supply your own keys here for due diligence testing diff --git a/exchanges/bitmex/bitmex_websocket.go b/exchanges/bitmex/bitmex_websocket.go index 69d473b3..b4f7c07d 100644 --- a/exchanges/bitmex/bitmex_websocket.go +++ b/exchanges/bitmex/bitmex_websocket.go @@ -13,7 +13,8 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -220,23 +221,22 @@ func (b *Bitmex) wsHandleIncomingData() { continue } - for _, trade := range trades.Data { + for i := range trades.Data { var timestamp time.Time - timestamp, err = time.Parse(time.RFC3339, trade.Timestamp) + timestamp, err = time.Parse(time.RFC3339, trades.Data[i].Timestamp) if err != nil { b.Websocket.DataHandler <- err continue } - // TODO: update this to support multiple asset types b.Websocket.DataHandler <- wshandler.TradeData{ Timestamp: timestamp, - Price: trade.Price, - Amount: float64(trade.Size), - CurrencyPair: currency.NewPairFromString(trade.Symbol), + Price: trades.Data[i].Price, + Amount: float64(trades.Data[i].Size), + CurrencyPair: currency.NewPairFromString(trades.Data[i].Symbol), Exchange: b.GetName(), AssetType: "CONTRACT", - Side: trade.Side, + Side: trades.Data[i].Side, } } @@ -326,99 +326,80 @@ func (b *Bitmex) wsHandleIncomingData() { } } -var snapshotloaded = make(map[currency.Pair]map[string]bool) - // ProcessOrderbook processes orderbook updates func (b *Bitmex) processOrderbook(data []OrderBookL2, action string, currencyPair currency.Pair, assetType string) error { // nolint: unparam if len(data) < 1 { return errors.New("bitmex_websocket.go error - no orderbook data") } - _, ok := snapshotloaded[currencyPair] - if !ok { - snapshotloaded[currencyPair] = make(map[string]bool) - } - - _, ok = snapshotloaded[currencyPair][assetType] - if !ok { - snapshotloaded[currencyPair][assetType] = false - } - switch action { case bitmexActionInitialData: - if !snapshotloaded[currencyPair][assetType] { - var newOrderBook orderbook.Base - var bids, asks []orderbook.Item - - for _, orderbookItem := range data { - if strings.EqualFold(orderbookItem.Side, exchange.SellOrderSide.ToString()) { - asks = append(asks, orderbook.Item{ - Price: orderbookItem.Price, - Amount: float64(orderbookItem.Size), - }) - continue - } - bids = append(bids, orderbook.Item{ - Price: orderbookItem.Price, - Amount: float64(orderbookItem.Size), + var newOrderBook orderbook.Base + var bids, asks []orderbook.Item + for i := range data { + if strings.EqualFold(data[i].Side, exchange.SellOrderSide.ToString()) { + asks = append(asks, orderbook.Item{ + Price: data[i].Price, + Amount: float64(data[i].Size), }) + continue } - - if len(bids) == 0 || len(asks) == 0 { - return errors.New("bitmex_websocket.go error - snapshot not initialised correctly") - } - - newOrderBook.Asks = asks - newOrderBook.Bids = bids - newOrderBook.AssetType = assetType - newOrderBook.Pair = currencyPair - - err := b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, b.GetName(), false) - if err != nil { - return fmt.Errorf("bitmex_websocket.go process orderbook error - %s", - err) - } - snapshotloaded[currencyPair][assetType] = true - b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ - Pair: currencyPair, - Asset: assetType, - Exchange: b.GetName(), - } + bids = append(bids, orderbook.Item{ + Price: data[i].Price, + Amount: float64(data[i].Size), + }) } + if len(bids) == 0 || len(asks) == 0 { + return errors.New("bitmex_websocket.go error - snapshot not initialised correctly") + } + + newOrderBook.Asks = asks + newOrderBook.Bids = bids + newOrderBook.AssetType = assetType + newOrderBook.Pair = currencyPair + err := b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false) + if err != nil { + return fmt.Errorf("bitmex_websocket.go process orderbook error - %s", + err) + } + b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ + Pair: currencyPair, + Asset: assetType, + Exchange: b.GetName(), + } default: - if snapshotloaded[currencyPair][assetType] { - var asks, bids []orderbook.Item - for _, orderbookItem := range data { - if orderbookItem.Side == "Sell" { - asks = append(asks, orderbook.Item{ - Price: orderbookItem.Price, - Amount: float64(orderbookItem.Size), - }) - continue - } - bids = append(bids, orderbook.Item{ - Price: orderbookItem.Price, - Amount: float64(orderbookItem.Size), + var asks, bids []orderbook.Item + for i := range data { + if strings.EqualFold(data[i].Side, "Sell") { + asks = append(asks, orderbook.Item{ + Price: data[i].Price, + Amount: float64(data[i].Size), }) + continue } + bids = append(bids, orderbook.Item{ + Price: data[i].Price, + Amount: float64(data[i].Size), + }) + } - err := b.Websocket.Orderbook.UpdateUsingID(bids, - asks, - currencyPair, - b.GetName(), - assetType, - action) + err := b.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: currencyPair, + UpdateTime: time.Now(), + AssetType: assetType, + Action: action, + }) + if err != nil { + return err + } - if err != nil { - return err - } - - b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ - Pair: currencyPair, - Asset: assetType, - Exchange: b.GetName(), - } + b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ + Pair: currencyPair, + Asset: assetType, + Exchange: b.GetName(), } } return nil diff --git a/exchanges/bitmex/bitmex_wrapper.go b/exchanges/bitmex/bitmex_wrapper.go index 94acc1a7..f40598e9 100644 --- a/exchanges/bitmex/bitmex_wrapper.go +++ b/exchanges/bitmex/bitmex_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bitstamp/bitstamp.go b/exchanges/bitstamp/bitstamp.go index b319b208..765d31a0 100644 --- a/exchanges/bitstamp/bitstamp.go +++ b/exchanges/bitstamp/bitstamp.go @@ -17,7 +17,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -94,6 +94,7 @@ func (b *Bitstamp) SetDefaults() { wshandler.WebsocketUnsubscribeSupported b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + b.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets configuration values to bitstamp @@ -158,6 +159,13 @@ func (b *Bitstamp) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + b.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + true, + true, + false, + exch.Name) } } diff --git a/exchanges/bitstamp/bitstamp_websocket.go b/exchanges/bitstamp/bitstamp_websocket.go index df8a412c..f9df0ce9 100644 --- a/exchanges/bitstamp/bitstamp_websocket.go +++ b/exchanges/bitstamp/bitstamp_websocket.go @@ -5,14 +5,14 @@ import ( "fmt" "net/http" "strconv" - "time" "github.com/gorilla/websocket" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -156,22 +156,21 @@ func (b *Bitstamp) Unsubscribe(channelToSubscribe wshandler.WebsocketChannelSubs return b.WebsocketConn.SendMessage(req) } -func (b *Bitstamp) wsUpdateOrderbook(ob websocketOrderBook, p currency.Pair, assetType string) error { - if len(ob.Asks) == 0 && len(ob.Bids) == 0 { +func (b *Bitstamp) wsUpdateOrderbook(update websocketOrderBook, p currency.Pair, assetType string) error { + if len(update.Asks) == 0 && len(update.Bids) == 0 { return errors.New("bitstamp_websocket.go error - no orderbook data") } var asks, bids []orderbook.Item - - if len(ob.Asks) > 0 { - for _, ask := range ob.Asks { - target, err := strconv.ParseFloat(ask[0], 64) + if len(update.Asks) > 0 { + for i := range update.Asks { + target, err := strconv.ParseFloat(update.Asks[i][0], 64) if err != nil { b.Websocket.DataHandler <- err continue } - amount, err := strconv.ParseFloat(ask[1], 64) + amount, err := strconv.ParseFloat(update.Asks[i][1], 64) if err != nil { b.Websocket.DataHandler <- err continue @@ -181,15 +180,15 @@ func (b *Bitstamp) wsUpdateOrderbook(ob websocketOrderBook, p currency.Pair, ass } } - if len(ob.Bids) > 0 { - for _, bid := range ob.Bids { - target, err := strconv.ParseFloat(bid[0], 64) + if len(update.Bids) > 0 { + for i := range update.Bids { + target, err := strconv.ParseFloat(update.Bids[i][0], 64) if err != nil { b.Websocket.DataHandler <- err continue } - amount, err := strconv.ParseFloat(bid[1], 64) + amount, err := strconv.ParseFloat(update.Bids[i][1], 64) if err != nil { b.Websocket.DataHandler <- err continue @@ -198,8 +197,13 @@ func (b *Bitstamp) wsUpdateOrderbook(ob websocketOrderBook, p currency.Pair, ass bids = append(bids, orderbook.Item{Price: target, Amount: amount}) } } - - err := b.Websocket.Orderbook.Update(bids, asks, p, time.Now(), b.GetName(), assetType) + err := b.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: p, + UpdateID: update.Timestamp, + AssetType: orderbook.Spot, + }) if err != nil { return err } @@ -224,17 +228,17 @@ func (b *Bitstamp) seedOrderBook() error { var newOrderBook orderbook.Base var asks, bids []orderbook.Item - for _, ask := range orderbookSeed.Asks { + for i := range orderbookSeed.Asks { var item orderbook.Item - item.Amount = ask.Amount - item.Price = ask.Price + item.Amount = orderbookSeed.Asks[i].Amount + item.Price = orderbookSeed.Asks[i].Price asks = append(asks, item) } - for _, bid := range orderbookSeed.Bids { + for i := range orderbookSeed.Bids { var item orderbook.Item - item.Amount = bid.Amount - item.Price = bid.Price + item.Amount = orderbookSeed.Bids[i].Amount + item.Price = orderbookSeed.Bids[i].Price bids = append(bids, item) } @@ -243,7 +247,7 @@ func (b *Bitstamp) seedOrderBook() error { newOrderBook.Pair = p[x] newOrderBook.AssetType = ticker.Spot - err = b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, b.GetName(), false) + err = b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false) if err != nil { return err } diff --git a/exchanges/bitstamp/bitstamp_wrapper.go b/exchanges/bitstamp/bitstamp_wrapper.go index 6d5eac13..85960e7d 100644 --- a/exchanges/bitstamp/bitstamp_wrapper.go +++ b/exchanges/bitstamp/bitstamp_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bittrex/bittrex.go b/exchanges/bittrex/bittrex.go index 060a25cd..6e071be0 100644 --- a/exchanges/bittrex/bittrex.go +++ b/exchanges/bittrex/bittrex.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/bittrex/bittrex_wrapper.go b/exchanges/bittrex/bittrex_wrapper.go index fe473492..adb24291 100644 --- a/exchanges/bittrex/bittrex_wrapper.go +++ b/exchanges/bittrex/bittrex_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/btcmarkets/btcmarkets.go b/exchanges/btcmarkets/btcmarkets.go index 11775c14..0c5701c7 100644 --- a/exchanges/btcmarkets/btcmarkets.go +++ b/exchanges/btcmarkets/btcmarkets.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/btcmarkets/btcmarkets_wrapper.go b/exchanges/btcmarkets/btcmarkets_wrapper.go index 11e7d8c9..4bfec6f6 100644 --- a/exchanges/btcmarkets/btcmarkets_wrapper.go +++ b/exchanges/btcmarkets/btcmarkets_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/btse/btse.go b/exchanges/btse/btse.go index f95fbe07..29de64a2 100644 --- a/exchanges/btse/btse.go +++ b/exchanges/btse/btse.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -71,6 +71,7 @@ func (b *BTSE) SetDefaults() { wshandler.WebsocketUnsubscribeSupported b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + b.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup takes in the supplied exchange configuration details and sets params @@ -129,6 +130,13 @@ func (b *BTSE) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + b.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + false, + false, + false, + false, + exch.Name) } } diff --git a/exchanges/btse/btse_websocket.go b/exchanges/btse/btse_websocket.go index 9f85d955..72ced158 100644 --- a/exchanges/btse/btse_websocket.go +++ b/exchanges/btse/btse_websocket.go @@ -11,7 +11,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -94,7 +94,7 @@ func (b *BTSE) WsHandleData() { b.Websocket.DataHandler <- wshandler.TickerData{ Timestamp: time.Now(), Pair: currency.NewPairDelimiter(t.ProductID, "-"), - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: b.GetName(), OpenPrice: price, } @@ -119,14 +119,14 @@ func (b *BTSE) WsHandleData() { // ProcessSnapshot processes the initial orderbook snap shot func (b *BTSE) wsProcessSnapshot(snapshot *websocketOrderbookSnapshot) error { var base orderbook.Base - for _, bid := range snapshot.Bids { - p := strings.Replace(bid[0].(string), ",", "", -1) + for i := range snapshot.Bids { + p := strings.Replace(snapshot.Bids[i][0].(string), ",", "", -1) price, err := strconv.ParseFloat(p, 64) if err != nil { return err } - a := strings.Replace(bid[1].(string), ",", "", -1) + a := strings.Replace(snapshot.Bids[i][1].(string), ",", "", -1) amount, err := strconv.ParseFloat(a, 64) if err != nil { return err @@ -136,14 +136,14 @@ func (b *BTSE) wsProcessSnapshot(snapshot *websocketOrderbookSnapshot) error { orderbook.Item{Price: price, Amount: amount}) } - for _, ask := range snapshot.Asks { - p := strings.Replace(ask[0].(string), ",", "", -1) + for i := range snapshot.Asks { + p := strings.Replace(snapshot.Asks[i][0].(string), ",", "", -1) price, err := strconv.ParseFloat(p, 64) if err != nil { return err } - a := strings.Replace(ask[1].(string), ",", "", -1) + a := strings.Replace(snapshot.Asks[i][1].(string), ",", "", -1) amount, err := strconv.ParseFloat(a, 64) if err != nil { return err @@ -154,19 +154,19 @@ func (b *BTSE) wsProcessSnapshot(snapshot *websocketOrderbookSnapshot) error { } p := currency.NewPairDelimiter(snapshot.ProductID, "-") - base.AssetType = "SPOT" + base.AssetType = orderbook.Spot base.Pair = p base.LastUpdated = time.Now() base.ExchangeName = b.Name - err := base.Process() + err := b.Websocket.Orderbook.LoadSnapshot(&base, true) if err != nil { return err } b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Pair: p, - Asset: "SPOT", + Asset: orderbook.Spot, Exchange: b.GetName(), } diff --git a/exchanges/btse/btse_wrapper.go b/exchanges/btse/btse_wrapper.go index db66e141..e02bcb04 100644 --- a/exchanges/btse/btse_wrapper.go +++ b/exchanges/btse/btse_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/coinbasepro/coinbasepro.go b/exchanges/coinbasepro/coinbasepro.go index 987171bd..d63fed4b 100644 --- a/exchanges/coinbasepro/coinbasepro.go +++ b/exchanges/coinbasepro/coinbasepro.go @@ -16,7 +16,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -95,6 +95,7 @@ func (c *CoinbasePro) SetDefaults() { wshandler.WebsocketSequenceNumberSupported c.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit c.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + c.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup initialises the exchange parameters with the current configuration @@ -158,6 +159,13 @@ func (c *CoinbasePro) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + c.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + true, + false, + false, + exch.Name) } } diff --git a/exchanges/coinbasepro/coinbasepro_test.go b/exchanges/coinbasepro/coinbasepro_test.go index f5fdba0b..2e9ff69c 100644 --- a/exchanges/coinbasepro/coinbasepro_test.go +++ b/exchanges/coinbasepro/coinbasepro_test.go @@ -10,7 +10,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) var c CoinbasePro diff --git a/exchanges/coinbasepro/coinbasepro_websocket.go b/exchanges/coinbasepro/coinbasepro_websocket.go index 0da53fc8..f411e424 100644 --- a/exchanges/coinbasepro/coinbasepro_websocket.go +++ b/exchanges/coinbasepro/coinbasepro_websocket.go @@ -12,7 +12,8 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" ) const ( @@ -88,7 +89,7 @@ func (c *CoinbasePro) WsHandleData() { c.Websocket.DataHandler <- wshandler.TickerData{ Timestamp: ticker.Time, Pair: currency.NewPairFromString(ticker.ProductID), - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: c.GetName(), OpenPrice: ticker.Open24H, HighPrice: ticker.High24H, @@ -177,13 +178,13 @@ func (c *CoinbasePro) WsHandleData() { // ProcessSnapshot processes the initial orderbook snap shot func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) error { var base orderbook.Base - for _, bid := range snapshot.Bids { - price, err := strconv.ParseFloat(bid[0].(string), 64) + for i := range snapshot.Bids { + price, err := strconv.ParseFloat(snapshot.Bids[i][0].(string), 64) if err != nil { return err } - amount, err := strconv.ParseFloat(bid[1].(string), 64) + amount, err := strconv.ParseFloat(snapshot.Bids[i][1].(string), 64) if err != nil { return err } @@ -192,13 +193,13 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro orderbook.Item{Price: price, Amount: amount}) } - for _, ask := range snapshot.Asks { - price, err := strconv.ParseFloat(ask[0].(string), 64) + for i := range snapshot.Asks { + price, err := strconv.ParseFloat(snapshot.Asks[i][0].(string), 64) if err != nil { return err } - amount, err := strconv.ParseFloat(ask[1].(string), 64) + amount, err := strconv.ParseFloat(snapshot.Asks[i][1].(string), 64) if err != nil { return err } @@ -208,17 +209,17 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro } pair := currency.NewPairFromString(snapshot.ProductID) - base.AssetType = "SPOT" + base.AssetType = orderbook.Spot base.Pair = pair - err := c.Websocket.Orderbook.LoadSnapshot(&base, c.GetName(), false) + err := c.Websocket.Orderbook.LoadSnapshot(&base, false) if err != nil { return err } c.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Pair: pair, - Asset: "SPOT", + Asset: orderbook.Spot, Exchange: c.GetName(), } @@ -227,33 +228,42 @@ func (c *CoinbasePro) ProcessSnapshot(snapshot *WebsocketOrderbookSnapshot) erro // ProcessUpdate updates the orderbook local cache func (c *CoinbasePro) ProcessUpdate(update WebsocketL2Update) error { - var Asks, Bids []orderbook.Item + var asks, bids []orderbook.Item - for _, data := range update.Changes { - price, _ := strconv.ParseFloat(data[1].(string), 64) - volume, _ := strconv.ParseFloat(data[2].(string), 64) + for i := range update.Changes { + price, _ := strconv.ParseFloat(update.Changes[i][1].(string), 64) + volume, _ := strconv.ParseFloat(update.Changes[i][2].(string), 64) - if data[0].(string) == "buy" { - Bids = append(Bids, orderbook.Item{Price: price, Amount: volume}) + if update.Changes[i][0].(string) == "buy" { + bids = append(bids, orderbook.Item{Price: price, Amount: volume}) } else { - Asks = append(Asks, orderbook.Item{Price: price, Amount: volume}) + asks = append(asks, orderbook.Item{Price: price, Amount: volume}) } } - if len(Asks) == 0 && len(Bids) == 0 { - return errors.New("coibasepro_websocket.go error - no data in websocket update") + if len(asks) == 0 && len(bids) == 0 { + return errors.New("coinbasepro_websocket.go error - no data in websocket update") } p := currency.NewPairFromString(update.ProductID) - - err := c.Websocket.Orderbook.Update(Bids, Asks, p, time.Now(), c.GetName(), "SPOT") + timestamp, err := time.Parse(time.RFC3339, update.Time) + if err != nil { + return err + } + err = c.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: p, + UpdateTime: timestamp, + AssetType: orderbook.Spot, + }) if err != nil { return err } c.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Pair: p, - Asset: "SPOT", + Asset: orderbook.Spot, Exchange: c.GetName(), } diff --git a/exchanges/coinbasepro/coinbasepro_wrapper.go b/exchanges/coinbasepro/coinbasepro_wrapper.go index 71abf011..5c0f02f2 100644 --- a/exchanges/coinbasepro/coinbasepro_wrapper.go +++ b/exchanges/coinbasepro/coinbasepro_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/coinut/coinut.go b/exchanges/coinut/coinut.go index b274690c..5a1c954d 100644 --- a/exchanges/coinut/coinut.go +++ b/exchanges/coinut/coinut.go @@ -12,9 +12,10 @@ import ( "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -86,6 +87,7 @@ func (c *COINUT) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported c.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit c.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + c.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets the current exchange configuration @@ -147,6 +149,13 @@ func (c *COINUT) Setup(exch *config.ExchangeConfig) { ResponseMaxLimit: exch.WebsocketResponseMaxLimit, RateLimit: coinutWebsocketRateLimit, } + c.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + true, + true, + false, + exch.Name) } } @@ -154,7 +163,7 @@ func (c *COINUT) Setup(exch *config.ExchangeConfig) { func (c *COINUT) GetInstruments() (Instruments, error) { var result Instruments params := make(map[string]interface{}) - params["sec_type"] = "SPOT" + params["sec_type"] = orderbook.Spot return result, c.SendHTTPRequest(coinutInstruments, params, false, &result) } diff --git a/exchanges/coinut/coinut_test.go b/exchanges/coinut/coinut_test.go index 89f04650..358c10d7 100644 --- a/exchanges/coinut/coinut_test.go +++ b/exchanges/coinut/coinut_test.go @@ -11,7 +11,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) var c COINUT diff --git a/exchanges/coinut/coinut_websocket.go b/exchanges/coinut/coinut_websocket.go index e385b243..262927b3 100644 --- a/exchanges/coinut/coinut_websocket.go +++ b/exchanges/coinut/coinut_websocket.go @@ -12,7 +12,8 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" ) const coinutWebsocketURL = "wss://wsapi.coinut.com" @@ -137,7 +138,7 @@ func (c *COINUT) wsProcessResponse(resp []byte) { c.Websocket.DataHandler <- wshandler.TickerData{ Timestamp: time.Unix(0, ticker.Timestamp), Exchange: c.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, HighPrice: ticker.HighestBuy, LowPrice: ticker.LowestSell, ClosePrice: ticker.Last, @@ -159,7 +160,7 @@ func (c *COINUT) wsProcessResponse(resp []byte) { currencyPair := instrumentListByCode[orderbooksnapshot.InstID] c.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: c.GetName(), - Asset: "SPOT", + Asset: orderbook.Spot, Pair: currency.NewPairFromString(currencyPair), } case "inst_order_book_update": @@ -177,7 +178,7 @@ func (c *COINUT) wsProcessResponse(resp []byte) { currencyPair := instrumentListByCode[orderbookUpdate.InstID] c.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: c.GetName(), - Asset: "SPOT", + Asset: orderbook.Spot, Pair: currency.NewPairFromString(currencyPair), } case "inst_trade": @@ -199,7 +200,7 @@ func (c *COINUT) wsProcessResponse(resp []byte) { c.Websocket.DataHandler <- wshandler.TradeData{ Timestamp: time.Unix(tradeUpdate.Timestamp, 0), CurrencyPair: currency.NewPairFromString(currencyPair), - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: c.GetName(), Price: tradeUpdate.Price, Side: tradeUpdate.Side, @@ -228,7 +229,7 @@ func (c *COINUT) GetNonce() int64 { func (c *COINUT) WsSetInstrumentList() error { request := wsRequest{ Request: "inst_list", - SecType: "SPOT", + SecType: orderbook.Spot, Nonce: c.WebsocketConn.GenerateMessageID(false), } resp, err := c.WebsocketConn.SendMessageReturnResponse(request.Nonce, request) @@ -253,18 +254,18 @@ func (c *COINUT) WsSetInstrumentList() error { // WsProcessOrderbookSnapshot processes the orderbook snapshot func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error { var bids []orderbook.Item - for _, bid := range ob.Buy { + for i := range ob.Buy { bids = append(bids, orderbook.Item{ - Amount: bid.Volume, - Price: bid.Price, + Amount: ob.Buy[i].Volume, + Price: ob.Buy[i].Price, }) } var asks []orderbook.Item - for _, ask := range ob.Sell { + for i := range ob.Sell { asks = append(asks, orderbook.Item{ - Amount: ask.Volume, - Price: ask.Price, + Amount: ob.Sell[i].Volume, + Price: ob.Sell[i].Price, }) } @@ -272,32 +273,25 @@ func (c *COINUT) WsProcessOrderbookSnapshot(ob *WsOrderbookSnapshot) error { newOrderBook.Asks = asks newOrderBook.Bids = bids newOrderBook.Pair = currency.NewPairFromString(instrumentListByCode[ob.InstID]) - newOrderBook.AssetType = "SPOT" + newOrderBook.AssetType = orderbook.Spot - return c.Websocket.Orderbook.LoadSnapshot(&newOrderBook, c.GetName(), false) + return c.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false) } // WsProcessOrderbookUpdate process an orderbook update -func (c *COINUT) WsProcessOrderbookUpdate(ob *WsOrderbookUpdate) error { - p := currency.NewPairFromString(instrumentListByCode[ob.InstID]) - - if ob.Side == "buy" { - return c.Websocket.Orderbook.Update([]orderbook.Item{ - {Price: ob.Price, Amount: ob.Volume}}, - nil, - p, - time.Now(), - c.GetName(), - "SPOT") +func (c *COINUT) WsProcessOrderbookUpdate(update *WsOrderbookUpdate) error { + p := currency.NewPairFromString(instrumentListByCode[update.InstID]) + bufferUpdate := &wsorderbook.WebsocketOrderbookUpdate{ + CurrencyPair: p, + UpdateID: update.TransID, + AssetType: orderbook.Spot, } - - return c.Websocket.Orderbook.Update([]orderbook.Item{ - {Price: ob.Price, Amount: ob.Volume}}, - nil, - p, - time.Now(), - c.GetName(), - "SPOT") + if strings.EqualFold(update.Side, "buy") { + bufferUpdate.Bids = []orderbook.Item{{Price: update.Price, Amount: update.Volume}} + } else { + bufferUpdate.Asks = []orderbook.Item{{Price: update.Price, Amount: update.Volume}} + } + return c.Websocket.Orderbook.Update(bufferUpdate) } // GenerateDefaultSubscriptions Adds default subscriptions to websocket to be handled by ManageSubscriptions() diff --git a/exchanges/coinut/coinut_wrapper.go b/exchanges/coinut/coinut_wrapper.go index 1f8b36ea..1ceed266 100644 --- a/exchanges/coinut/coinut_wrapper.go +++ b/exchanges/coinut/coinut_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/exchange.go b/exchanges/exchange.go index 4836835f..8162182e 100644 --- a/exchanges/exchange.go +++ b/exchanges/exchange.go @@ -16,7 +16,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -32,6 +32,8 @@ const ( DefaultWebsocketResponseCheckTimeout = time.Millisecond * 30 // DefaultWebsocketResponseMaxLimit is the default max wait for an expected websocket response before a timeout DefaultWebsocketResponseMaxLimit = time.Second * 7 + // DefaultWebsocketOrderbookBufferLimit is the maximum number of orderbook updates that get stored before being applied + DefaultWebsocketOrderbookBufferLimit = 5 ) // FeeType custom type for calculating fees based on method @@ -267,6 +269,7 @@ type Base struct { RESTPollingDelay time.Duration WebsocketResponseCheckTimeout time.Duration WebsocketResponseMaxLimit time.Duration + WebsocketOrderbookBufferLimit int64 AuthenticatedAPISupport bool AuthenticatedWebsocketAPISupport bool APIWithdrawPermissions uint32 diff --git a/exchanges/exchange_test.go b/exchanges/exchange_test.go index a06eb534..1aec6d20 100644 --- a/exchanges/exchange_test.go +++ b/exchanges/exchange_test.go @@ -9,9 +9,10 @@ import ( "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) const ( @@ -217,7 +218,7 @@ func TestSetAssetTypes(t *testing.T) { } b.Name = defaultTestExchange - b.AssetTypes = []string{"SPOT"} + b.AssetTypes = []string{orderbook.Spot} err = b.SetAssetTypes() if err != nil { t.Fatalf("Test failed. TestSetAssetTypes. Error %s", err) @@ -255,7 +256,7 @@ func TestSetAssetTypes(t *testing.T) { func TestGetAssetTypes(t *testing.T) { testExchange := Base{ - AssetTypes: []string{"SPOT", "Binary", "Futures"}, + AssetTypes: []string{orderbook.Spot, "Binary", "Futures"}, } aT := testExchange.GetAssetTypes() diff --git a/exchanges/exmo/exmo.go b/exchanges/exmo/exmo.go index ce209e3c..a4694a2f 100644 --- a/exchanges/exmo/exmo.go +++ b/exchanges/exmo/exmo.go @@ -15,7 +15,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/exmo/exmo_wrapper.go b/exchanges/exmo/exmo_wrapper.go index 6b6ac72b..5502096e 100644 --- a/exchanges/exmo/exmo_wrapper.go +++ b/exchanges/exmo/exmo_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/gateio/gateio.go b/exchanges/gateio/gateio.go index 64e361d9..c3b55dfe 100644 --- a/exchanges/gateio/gateio.go +++ b/exchanges/gateio/gateio.go @@ -15,7 +15,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -85,6 +85,7 @@ func (g *Gateio) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported g.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit g.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + g.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets user configuration @@ -147,6 +148,13 @@ func (g *Gateio) Setup(exch *config.ExchangeConfig) { ResponseMaxLimit: exch.WebsocketResponseMaxLimit, RateLimit: gateioWebsocketRateLimit, } + g.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + false, + false, + false, + exch.Name) } } diff --git a/exchanges/gateio/gateio_test.go b/exchanges/gateio/gateio_test.go index 49a05474..8f2b418b 100644 --- a/exchanges/gateio/gateio_test.go +++ b/exchanges/gateio/gateio_test.go @@ -10,7 +10,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please supply your own APIKEYS here for due diligence testing diff --git a/exchanges/gateio/gateio_websocket.go b/exchanges/gateio/gateio_websocket.go index 946bfe83..6334450b 100644 --- a/exchanges/gateio/gateio_websocket.go +++ b/exchanges/gateio/gateio_websocket.go @@ -13,7 +13,8 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -135,7 +136,7 @@ func (g *Gateio) WsHandleData() { g.Websocket.DataHandler <- wshandler.TickerData{ Timestamp: time.Now(), Pair: currency.NewPairFromString(c), - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: g.GetName(), ClosePrice: ticker.Close, Quantity: ticker.BaseVolume, @@ -159,15 +160,15 @@ func (g *Gateio) WsHandleData() { continue } - for _, trade := range trades { + for i := range trades { g.Websocket.DataHandler <- wshandler.TradeData{ Timestamp: time.Now(), CurrencyPair: currency.NewPairFromString(c), - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: g.GetName(), - Price: trade.Price, - Amount: trade.Amount, - Side: trade.Type, + Price: trades[i].Price, + Amount: trades[i].Amount, + Side: trades[i].Type, } } @@ -196,9 +197,9 @@ func (g *Gateio) WsHandleData() { var asks, bids []orderbook.Item askData, askOk := data["asks"] - for _, ask := range askData { - amount, _ := strconv.ParseFloat(ask[1], 64) - price, _ := strconv.ParseFloat(ask[0], 64) + for i := range askData { + amount, _ := strconv.ParseFloat(askData[i][1], 64) + price, _ := strconv.ParseFloat(askData[i][0], 64) asks = append(asks, orderbook.Item{ Amount: amount, Price: price, @@ -206,9 +207,9 @@ func (g *Gateio) WsHandleData() { } bidData, bidOk := data["bids"] - for _, bid := range bidData { - amount, _ := strconv.ParseFloat(bid[1], 64) - price, _ := strconv.ParseFloat(bid[0], 64) + for i := range bidData { + amount, _ := strconv.ParseFloat(bidData[i][1], 64) + price, _ := strconv.ParseFloat(bidData[i][0], 64) bids = append(bids, orderbook.Item{ Amount: amount, Price: price, @@ -231,22 +232,23 @@ func (g *Gateio) WsHandleData() { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = "SPOT" + newOrderBook.AssetType = orderbook.Spot newOrderBook.Pair = currency.NewPairFromString(c) err = g.Websocket.Orderbook.LoadSnapshot(&newOrderBook, - g.GetName(), false) if err != nil { g.Websocket.DataHandler <- err } } else { - err = g.Websocket.Orderbook.Update(asks, - bids, - currency.NewPairFromString(c), - time.Now(), - g.GetName(), - "SPOT") + err = g.Websocket.Orderbook.Update( + &wsorderbook.WebsocketOrderbookUpdate{ + Asks: asks, + Bids: bids, + CurrencyPair: currency.NewPairFromString(c), + UpdateTime: time.Now(), + AssetType: orderbook.Spot, + }) if err != nil { g.Websocket.DataHandler <- err } @@ -254,7 +256,7 @@ func (g *Gateio) WsHandleData() { g.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Pair: currency.NewPairFromString(c), - Asset: "SPOT", + Asset: orderbook.Spot, Exchange: g.GetName(), } @@ -275,7 +277,7 @@ func (g *Gateio) WsHandleData() { g.Websocket.DataHandler <- wshandler.KlineData{ Timestamp: time.Now(), Pair: currency.NewPairFromString(data[7].(string)), - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: g.GetName(), OpenPrice: open, ClosePrice: closePrice, @@ -334,8 +336,8 @@ func (g *Gateio) GenerateDefaultSubscriptions() { // Subscribe sends a websocket message to receive data from the channel func (g *Gateio) Subscribe(channelToSubscribe wshandler.WebsocketChannelSubscription) error { params := []interface{}{channelToSubscribe.Currency.String()} - for _, paramValue := range channelToSubscribe.Params { - params = append(params, paramValue) + for i := range channelToSubscribe.Params { + params = append(params, channelToSubscribe.Params[i]) } subscribe := WebsocketRequest{ diff --git a/exchanges/gateio/gateio_wrapper.go b/exchanges/gateio/gateio_wrapper.go index a656d016..9cbfbaef 100644 --- a/exchanges/gateio/gateio_wrapper.go +++ b/exchanges/gateio/gateio_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/gemini/gemini.go b/exchanges/gemini/gemini.go index 91886898..a69d11a9 100644 --- a/exchanges/gemini/gemini.go +++ b/exchanges/gemini/gemini.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -116,8 +116,8 @@ func (g *Gemini) SetDefaults() { g.SupportsAutoPairUpdating = true g.SupportsRESTTickerBatching = false g.Requester = request.New(g.Name, - request.NewRateLimit(time.Minute, geminiAuthRate), - request.NewRateLimit(time.Minute, geminiUnauthRate), + request.NewRateLimit(time.Second, geminiAuthRate), + request.NewRateLimit(time.Second, geminiUnauthRate), common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) g.APIUrlDefault = geminiAPIURL g.APIUrl = g.APIUrlDefault @@ -128,6 +128,7 @@ func (g *Gemini) SetDefaults() { wshandler.WebsocketSequenceNumberSupported g.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit g.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + g.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets exchange configuration parameters @@ -186,6 +187,13 @@ func (g *Gemini) Setup(exch *config.ExchangeConfig) { } responseCheckTimeout = exch.WebsocketResponseCheckTimeout responseMaxLimit = exch.WebsocketResponseMaxLimit + g.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + true, + false, + false, + exch.Name) } } diff --git a/exchanges/gemini/gemini_test.go b/exchanges/gemini/gemini_test.go index e85e4291..54eef877 100644 --- a/exchanges/gemini/gemini_test.go +++ b/exchanges/gemini/gemini_test.go @@ -11,7 +11,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please enter sandbox API keys & assigned roles for better testing procedures diff --git a/exchanges/gemini/gemini_types.go b/exchanges/gemini/gemini_types.go index bd256a73..206c9cbe 100644 --- a/exchanges/gemini/gemini_types.go +++ b/exchanges/gemini/gemini_types.go @@ -212,7 +212,7 @@ type WsMarketUpdateResponse struct { // Event defines orderbook and trade data type Event struct { - Type string `json:"change"` + Type string `json:"type"` Reason string `json:"reason"` Price float64 `json:"price,string"` Delta float64 `json:"delta,string"` diff --git a/exchanges/gemini/gemini_websocket.go b/exchanges/gemini/gemini_websocket.go index 988c18aa..bc1691d9 100644 --- a/exchanges/gemini/gemini_websocket.go +++ b/exchanges/gemini/gemini_websocket.go @@ -14,7 +14,8 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -110,9 +111,11 @@ func (g *Gemini) WsSecureSubscribe(dialer *websocket.Dialer, url string) error { headers.Add("Cache-Control", "no-cache") g.AuthenticatedWebsocketConn = &wshandler.WebsocketConnection{ - ExchangeName: g.Name, - URL: endpoint, - Verbose: g.Verbose, + ExchangeName: g.Name, + URL: endpoint, + Verbose: g.Verbose, + ResponseCheckTimeout: responseCheckTimeout, + ResponseMaxLimit: responseMaxLimit, } err = g.AuthenticatedWebsocketConn.Dial(dialer, headers) if err != nil { @@ -253,86 +256,75 @@ func (g *Gemini) WsHandleData() { func (g *Gemini) wsProcessUpdate(result WsMarketUpdateResponse, pair currency.Pair) { if result.Timestamp == 0 && result.TimestampMS == 0 { var bids, asks []orderbook.Item - for _, event := range result.Events { - if event.Reason != "initial" { + for i := range result.Events { + if result.Events[i].Reason != "initial" { g.Websocket.DataHandler <- errors.New("gemini_websocket.go orderbook should be snapshot only") continue } - - if event.Side == "ask" { + if result.Events[i].Side == "ask" { asks = append(asks, orderbook.Item{ - Amount: event.Remaining, - Price: event.Price, + Amount: result.Events[i].Remaining, + Price: result.Events[i].Price, }) } else { bids = append(bids, orderbook.Item{ - Amount: event.Remaining, - Price: event.Price, + Amount: result.Events[i].Remaining, + Price: result.Events[i].Price, }) } } - var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = "SPOT" + newOrderBook.AssetType = orderbook.Spot newOrderBook.Pair = pair - err := g.Websocket.Orderbook.LoadSnapshot(&newOrderBook, - g.GetName(), false) if err != nil { g.Websocket.DataHandler <- err return } - g.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: pair, - Asset: "SPOT", + Asset: orderbook.Spot, Exchange: g.GetName()} } else { - for _, event := range result.Events { - if event.Type == "trade" { + var asks, bids []orderbook.Item + for i := 0; i < len(result.Events); i++ { + if result.Events[i].Type == "trade" { g.Websocket.DataHandler <- wshandler.TradeData{ Timestamp: time.Now(), CurrencyPair: pair, - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: g.Name, EventTime: result.Timestamp, - Price: event.Price, - Amount: event.Amount, - Side: event.MakerSide, + Price: result.Events[i].Price, + Amount: result.Events[i].Amount, + Side: result.Events[i].MakerSide, } - } else { - var i orderbook.Item - i.Amount = event.Remaining - i.Price = event.Price - if event.Side == "ask" { - err := g.Websocket.Orderbook.Update(nil, - []orderbook.Item{i}, - pair, - time.Now(), - g.GetName(), - "SPOT") - if err != nil { - g.Websocket.DataHandler <- err - } + item := orderbook.Item{ + Amount: result.Events[i].Remaining, + Price: result.Events[i].Price, + } + if result.Events[i].Side == "ask" { + asks = append(asks, item) } else { - err := g.Websocket.Orderbook.Update([]orderbook.Item{i}, - nil, - pair, - time.Now(), - g.GetName(), - "SPOT") - if err != nil { - g.Websocket.DataHandler <- err - } + bids = append(bids, item) } } } - + err := g.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{ + Asks: asks, + Bids: bids, + CurrencyPair: pair, + UpdateTime: time.Unix(0, result.TimestampMS), + AssetType: orderbook.Spot, + }) + if err != nil { + g.Websocket.DataHandler <- fmt.Errorf("%v %v", g.Name, err) + } g.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{Pair: pair, - Asset: "SPOT", + Asset: orderbook.Spot, Exchange: g.GetName()} } } diff --git a/exchanges/gemini/gemini_wrapper.go b/exchanges/gemini/gemini_wrapper.go index 8611199b..0c767178 100644 --- a/exchanges/gemini/gemini_wrapper.go +++ b/exchanges/gemini/gemini_wrapper.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/hitbtc/hitbtc.go b/exchanges/hitbtc/hitbtc.go index f0a6d745..4b9b791f 100644 --- a/exchanges/hitbtc/hitbtc.go +++ b/exchanges/hitbtc/hitbtc.go @@ -15,7 +15,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -89,6 +89,7 @@ func (h *HitBTC) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported h.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit h.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + h.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets user exchange configuration settings @@ -150,6 +151,13 @@ func (h *HitBTC) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + h.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + true, + true, + false, + exch.Name) } } diff --git a/exchanges/hitbtc/hitbtc_test.go b/exchanges/hitbtc/hitbtc_test.go index 42083f91..8860ef9e 100644 --- a/exchanges/hitbtc/hitbtc_test.go +++ b/exchanges/hitbtc/hitbtc_test.go @@ -11,7 +11,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) var h HitBTC diff --git a/exchanges/hitbtc/hitbtc_websocket.go b/exchanges/hitbtc/hitbtc_websocket.go index 77832798..67703c1f 100644 --- a/exchanges/hitbtc/hitbtc_websocket.go +++ b/exchanges/hitbtc/hitbtc_websocket.go @@ -13,7 +13,8 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/nonce" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -114,7 +115,7 @@ func (h *HitBTC) handleSubscriptionUpdates(resp wshandler.WebsocketResponse, ini } h.Websocket.DataHandler <- wshandler.TickerData{ Exchange: h.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, Pair: currency.NewPairFromString(ticker.Params.Symbol), Quantity: ticker.Params.Volume, Timestamp: ts, @@ -225,13 +226,13 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob WsOrderbook) error { } var bids []orderbook.Item - for _, bid := range ob.Params.Bid { - bids = append(bids, orderbook.Item{Amount: bid.Size, Price: bid.Price}) + for i := range ob.Params.Bid { + bids = append(bids, orderbook.Item{Amount: ob.Params.Bid[i].Size, Price: ob.Params.Bid[i].Price}) } var asks []orderbook.Item - for _, ask := range ob.Params.Ask { - asks = append(asks, orderbook.Item{Amount: ask.Size, Price: ask.Price}) + for i := range ob.Params.Ask { + asks = append(asks, orderbook.Item{Amount: ob.Params.Ask[i].Size, Price: ob.Params.Ask[i].Price}) } p := currency.NewPairFromString(ob.Params.Symbol) @@ -239,17 +240,17 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob WsOrderbook) error { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = "SPOT" + newOrderBook.AssetType = orderbook.Spot newOrderBook.Pair = p - err := h.Websocket.Orderbook.LoadSnapshot(&newOrderBook, h.GetName(), false) + err := h.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false) if err != nil { return err } h.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: h.GetName(), - Asset: "SPOT", + Asset: orderbook.Spot, Pair: p, } @@ -257,30 +258,35 @@ func (h *HitBTC) WsProcessOrderbookSnapshot(ob WsOrderbook) error { } // WsProcessOrderbookUpdate updates a local cache -func (h *HitBTC) WsProcessOrderbookUpdate(ob WsOrderbook) error { - if len(ob.Params.Bid) == 0 && len(ob.Params.Ask) == 0 { +func (h *HitBTC) WsProcessOrderbookUpdate(update WsOrderbook) error { + if len(update.Params.Bid) == 0 && len(update.Params.Ask) == 0 { return errors.New("hitbtc_websocket.go error - no data") } var bids, asks []orderbook.Item - for _, bid := range ob.Params.Bid { - bids = append(bids, orderbook.Item{Price: bid.Price, Amount: bid.Size}) + for i := range update.Params.Bid { + bids = append(bids, orderbook.Item{Price: update.Params.Bid[i].Price, Amount: update.Params.Bid[i].Size}) } - for _, ask := range ob.Params.Ask { - asks = append(asks, orderbook.Item{Price: ask.Price, Amount: ask.Size}) + for i := range update.Params.Ask { + asks = append(asks, orderbook.Item{Price: update.Params.Ask[i].Price, Amount: update.Params.Ask[i].Size}) } - p := currency.NewPairFromString(ob.Params.Symbol) - - err := h.Websocket.Orderbook.Update(bids, asks, p, time.Now(), h.GetName(), "SPOT") + p := currency.NewPairFromString(update.Params.Symbol) + err := h.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{ + Asks: asks, + Bids: bids, + CurrencyPair: p, + UpdateID: update.Params.Sequence, + AssetType: orderbook.Spot, + }) if err != nil { return err } h.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: h.GetName(), - Asset: "SPOT", + Asset: orderbook.Spot, Pair: p, } return nil diff --git a/exchanges/hitbtc/hitbtc_wrapper.go b/exchanges/hitbtc/hitbtc_wrapper.go index 2d40180a..105b3647 100644 --- a/exchanges/hitbtc/hitbtc_wrapper.go +++ b/exchanges/hitbtc/hitbtc_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/huobi/huobi.go b/exchanges/huobi/huobi.go index 68a9a598..19351e0f 100644 --- a/exchanges/huobi/huobi.go +++ b/exchanges/huobi/huobi.go @@ -22,7 +22,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -105,6 +105,7 @@ func (h *HUOBI) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported h.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit h.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + h.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets user configuration @@ -177,6 +178,13 @@ func (h *HUOBI) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + h.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + false, + false, + false, + false, + exch.Name) } } diff --git a/exchanges/huobi/huobi_test.go b/exchanges/huobi/huobi_test.go index d46d731e..dd2f3452 100644 --- a/exchanges/huobi/huobi_test.go +++ b/exchanges/huobi/huobi_test.go @@ -16,7 +16,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please supply you own test keys here for due diligence testing. diff --git a/exchanges/huobi/huobi_websocket.go b/exchanges/huobi/huobi_websocket.go index 216417c1..2000dfb1 100644 --- a/exchanges/huobi/huobi_websocket.go +++ b/exchanges/huobi/huobi_websocket.go @@ -13,7 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -242,7 +242,7 @@ func (h *HUOBI) wsHandleMarketData(resp WsMessage) { h.Websocket.DataHandler <- wshandler.KlineData{ Timestamp: time.Unix(0, kline.Timestamp), Exchange: h.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, Pair: currency.NewPairFromString(data[1]), OpenPrice: kline.Tick.Open, ClosePrice: kline.Tick.Close, @@ -260,7 +260,7 @@ func (h *HUOBI) wsHandleMarketData(resp WsMessage) { data := common.SplitStrings(trade.Channel, ".") h.Websocket.DataHandler <- wshandler.TradeData{ Exchange: h.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, CurrencyPair: currency.NewPairFromString(data[1]), Timestamp: time.Unix(0, trade.Tick.Timestamp), } @@ -268,37 +268,31 @@ func (h *HUOBI) wsHandleMarketData(resp WsMessage) { } // WsProcessOrderbook processes new orderbook data -func (h *HUOBI) WsProcessOrderbook(ob *WsDepth, symbol string) error { - var bids []orderbook.Item - for _, data := range ob.Tick.Bids { - bidLevel := data.([]interface{}) +func (h *HUOBI) WsProcessOrderbook(update *WsDepth, symbol string) error { + p := currency.NewPairFromString(symbol) + var bids, asks []orderbook.Item + for i := 0; i < len(update.Tick.Bids); i++ { + bidLevel := update.Tick.Bids[i].([]interface{}) bids = append(bids, orderbook.Item{Price: bidLevel[0].(float64), Amount: bidLevel[0].(float64)}) } - - var asks []orderbook.Item - for _, data := range ob.Tick.Asks { - askLevel := data.([]interface{}) + for i := 0; i < len(update.Tick.Asks); i++ { + askLevel := update.Tick.Asks[i].([]interface{}) asks = append(asks, orderbook.Item{Price: askLevel[0].(float64), Amount: askLevel[0].(float64)}) } - - p := currency.NewPairFromString(symbol) - var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids newOrderBook.Pair = p - - err := h.Websocket.Orderbook.LoadSnapshot(&newOrderBook, h.GetName(), false) + err := h.Websocket.Orderbook.LoadSnapshot(&newOrderBook, true) if err != nil { return err } - h.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Pair: p, Exchange: h.GetName(), - Asset: "SPOT", + Asset: orderbook.Spot, } return nil diff --git a/exchanges/huobi/huobi_wrapper.go b/exchanges/huobi/huobi_wrapper.go index a59e9b52..752a6baf 100644 --- a/exchanges/huobi/huobi_wrapper.go +++ b/exchanges/huobi/huobi_wrapper.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/huobihadax/huobihadax.go b/exchanges/huobihadax/huobihadax.go index d5a89398..1b6120cd 100644 --- a/exchanges/huobihadax/huobihadax.go +++ b/exchanges/huobihadax/huobihadax.go @@ -16,7 +16,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -100,6 +100,7 @@ func (h *HUOBIHADAX) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported h.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit h.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + h.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } @@ -172,6 +173,13 @@ func (h *HUOBIHADAX) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + h.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + false, + false, + false, + false, + exch.Name) } } diff --git a/exchanges/huobihadax/huobihadax_test.go b/exchanges/huobihadax/huobihadax_test.go index a4224756..f6ea96e3 100644 --- a/exchanges/huobihadax/huobihadax_test.go +++ b/exchanges/huobihadax/huobihadax_test.go @@ -11,7 +11,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please supply your own APIKEYS here for due diligence testing diff --git a/exchanges/huobihadax/huobihadax_websocket.go b/exchanges/huobihadax/huobihadax_websocket.go index 947947b0..dbb70159 100644 --- a/exchanges/huobihadax/huobihadax_websocket.go +++ b/exchanges/huobihadax/huobihadax_websocket.go @@ -13,7 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -244,7 +244,7 @@ func (h *HUOBIHADAX) wsHandleMarketData(resp WsMessage) { h.Websocket.DataHandler <- wshandler.KlineData{ Timestamp: time.Unix(0, kline.Timestamp), Exchange: h.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, Pair: currency.NewPairFromString(data[1]), OpenPrice: kline.Tick.Open, ClosePrice: kline.Tick.Close, @@ -262,7 +262,7 @@ func (h *HUOBIHADAX) wsHandleMarketData(resp WsMessage) { data := common.SplitStrings(trade.Channel, ".") h.Websocket.DataHandler <- wshandler.TradeData{ Exchange: h.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, CurrencyPair: currency.NewPairFromString(data[1]), Timestamp: time.Unix(0, trade.Tick.Timestamp), } @@ -270,37 +270,31 @@ func (h *HUOBIHADAX) wsHandleMarketData(resp WsMessage) { } // WsProcessOrderbook processes new orderbook data -func (h *HUOBIHADAX) WsProcessOrderbook(ob *WsDepth, symbol string) error { - var bids []orderbook.Item - for _, data := range ob.Tick.Bids { - bidLevel := data.([]interface{}) +func (h *HUOBIHADAX) WsProcessOrderbook(update *WsDepth, symbol string) error { + p := currency.NewPairFromString(symbol) + var bids, asks []orderbook.Item + for i := 0; i < len(update.Tick.Bids); i++ { + bidLevel := update.Tick.Bids[i].([]interface{}) bids = append(bids, orderbook.Item{Price: bidLevel[0].(float64), Amount: bidLevel[0].(float64)}) } - - var asks []orderbook.Item - for _, data := range ob.Tick.Asks { - askLevel := data.([]interface{}) + for i := 0; i < len(update.Tick.Asks); i++ { + askLevel := update.Tick.Asks[i].([]interface{}) asks = append(asks, orderbook.Item{Price: askLevel[0].(float64), Amount: askLevel[0].(float64)}) } - - p := currency.NewPairFromString(symbol) - var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids newOrderBook.Pair = p - - err := h.Websocket.Orderbook.LoadSnapshot(&newOrderBook, h.GetName(), false) + err := h.Websocket.Orderbook.LoadSnapshot(&newOrderBook, true) if err != nil { return err } - h.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Pair: p, Exchange: h.GetName(), - Asset: "SPOT", + Asset: orderbook.Spot, } return nil diff --git a/exchanges/huobihadax/huobihadax_wrapper.go b/exchanges/huobihadax/huobihadax_wrapper.go index 6ff3a89b..37c883f1 100644 --- a/exchanges/huobihadax/huobihadax_wrapper.go +++ b/exchanges/huobihadax/huobihadax_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/itbit/itbit.go b/exchanges/itbit/itbit.go index 976073fd..1a1019df 100644 --- a/exchanges/itbit/itbit.go +++ b/exchanges/itbit/itbit.go @@ -16,7 +16,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/itbit/itbit_wrapper.go b/exchanges/itbit/itbit_wrapper.go index cfc8ea10..a8dce7b0 100644 --- a/exchanges/itbit/itbit_wrapper.go +++ b/exchanges/itbit/itbit_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/kraken/kraken.go b/exchanges/kraken/kraken.go index 4963d77f..539ba36a 100644 --- a/exchanges/kraken/kraken.go +++ b/exchanges/kraken/kraken.go @@ -16,7 +16,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -100,7 +100,7 @@ func (k *Kraken) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported k.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit k.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout - + k.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets current exchange configuration @@ -161,6 +161,13 @@ func (k *Kraken) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + k.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + true, + false, + false, + exch.Name) } } diff --git a/exchanges/kraken/kraken_test.go b/exchanges/kraken/kraken_test.go index 47c521a7..a2186444 100644 --- a/exchanges/kraken/kraken_test.go +++ b/exchanges/kraken/kraken_test.go @@ -1,18 +1,15 @@ package kraken import ( - "fmt" "net/http" - "strings" "testing" "github.com/gorilla/websocket" - "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) var k Kraken @@ -647,107 +644,6 @@ func TestWithdrawCancel(t *testing.T) { // ---------------------------- Websocket tests ----------------------------------------- -// TestOrderbookBufferReset websocket test -func TestOrderbookBufferReset(t *testing.T) { - if k.Name == "" { - k.SetDefaults() - TestSetup(t) - } - if !k.Websocket.IsEnabled() { - t.Skip("Websocket not enabled, skipping") - } - var obUpdates []string - obpartial := `[0,{"as":[["5541.30000","2.50700000","0"]],"bs":[["5541.20000","1.52900000","0"]]}]` - for i := 1; i < orderbookBufferLimit+2; i++ { - obUpdates = append(obUpdates, fmt.Sprintf(`[0,{"a":[["5541.30000","2.50700000","%v"]],"b":[["5541.30000","1.00000000","%v"]]}]`, i, i)) - } - k.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride() - var dataResponse WebsocketDataResponse - err := common.JSONDecode([]byte(obpartial), &dataResponse) - if err != nil { - t.Errorf("Could not parse, %v", err) - } - obData := dataResponse[1].(map[string]interface{}) - channelData := WebsocketChannelData{ - ChannelID: 0, - Subscription: "orderbook", - Pair: currency.NewPairWithDelimiter("XBT", "USD", "/"), - } - - k.wsProcessOrderBookPartial( - &channelData, - obData, - ) - - for i := 0; i < len(obUpdates); i++ { - err = common.JSONDecode([]byte(obUpdates[i]), &dataResponse) - if err != nil { - t.Errorf("Could not parse, %v", err) - } - obData = dataResponse[1].(map[string]interface{}) - if i < len(obUpdates)-1 { - k.wsProcessOrderBookBuffer(&channelData, obData) - } else if i == len(obUpdates)-1 { - k.wsProcessOrderBookUpdate(&channelData) - k.wsProcessOrderBookBuffer(&channelData, obData) - if len(orderbookBuffer[channelData.ChannelID]) != 1 { - t.Error("Buffer should have 1 entry after being reset") - } - } - } -} - -// TestOrderbookBufferReset websocket test -func TestOrderBookOutOfOrder(t *testing.T) { - if k.Name == "" { - k.SetDefaults() - TestSetup(t) - } - if !k.Websocket.IsEnabled() { - t.Skip("Websocket not enabled, skipping") - } - obpartial := `[0,{"as":[["5541.30000","2.50700000","0"]],"bs":[["5541.20000","1.52900000","5"]]}]` - obupdate1 := `[0,{"a":[["5541.30000","0.00000000","1"]],"b":[["5541.30000","0.00000000","3"]]}]` - obupdate2 := `[0,{"a":[["5541.30000","2.50700000","2"]],"b":[["5541.30000","0.00000000","1"]]}]` - - k.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride() - var dataResponse WebsocketDataResponse - err := common.JSONDecode([]byte(obpartial), &dataResponse) - if err != nil { - t.Errorf("Could not parse, %v", err) - } - obData := dataResponse[1].(map[string]interface{}) - channelData := WebsocketChannelData{ - ChannelID: 0, - Subscription: "orderbook", - Pair: currency.NewPairWithDelimiter("XBT", "USD", "/"), - } - - k.wsProcessOrderBookPartial( - &channelData, - obData, - ) - - err = common.JSONDecode([]byte(obupdate1), &dataResponse) - if err != nil { - t.Errorf("Could not parse, %v", err) - } - obData = dataResponse[1].(map[string]interface{}) - k.wsProcessOrderBookBuffer(&channelData, obData) - - err = common.JSONDecode([]byte(obupdate2), &dataResponse) - if err != nil { - t.Errorf("Could not parse, %v", err) - } - obData = dataResponse[1].(map[string]interface{}) - k.wsProcessOrderBookBuffer(&channelData, obData) - - err = k.wsProcessOrderBookUpdate(&channelData) - if !strings.Contains(err.Error(), "orderbook update out of order") { - t.Error("Expected out of order orderbook error") - } -} - func setupWsTests(t *testing.T) { if wsSetupRan { return diff --git a/exchanges/kraken/kraken_websocket.go b/exchanges/kraken/kraken_websocket.go index 1a853dcd..dc53c3a0 100644 --- a/exchanges/kraken/kraken_websocket.go +++ b/exchanges/kraken/kraken_websocket.go @@ -5,16 +5,15 @@ import ( "fmt" "math" "net/http" - "sort" "strconv" - "sync" "time" "github.com/gorilla/websocket" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/currency" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -40,21 +39,13 @@ const ( krakenWsSpread = "spread" krakenWsOrderbook = "book" // Only supported asset type + krakenWsAssetType = orderbook.Spot orderbookBufferLimit = 3 krakenWsRateLimit = 50 ) // orderbookMutex Ensures if two entries arrive at once, only one can be processed at a time -var orderbookMutex sync.Mutex var subscriptionChannelPair []WebsocketChannelData - -// krakenOrderBooks TODO THIS IS A TEMPORARY SOLUTION UNTIL ENGINE BRANCH IS MERGED -// WS orderbook data can only rely on WS orderbook data -// Currently REST and WS runs simultaneously, dirtying the data -var krakenOrderBooks map[int64]orderbook.Base - -// orderbookBuffer Stores orderbook updates per channel -var orderbookBuffer map[int64][]orderbook.Base var subscribeToDefaultChannels = true // Channels require a topic and a currency @@ -227,22 +218,6 @@ func (k *Kraken) WsHandleEventResponse(response *WebsocketEventResponse, rawResp // addNewSubscriptionChannelData stores channel ids, pairs and subscription types to an array // allowing correlation between subscriptions and returned data func addNewSubscriptionChannelData(response *WebsocketEventResponse) { - for i := range subscriptionChannelPair { - if response.ChannelID != subscriptionChannelPair[i].ChannelID { - continue - } - // kill the stale orderbooks due to resubscribing - if orderbookBuffer == nil { - orderbookBuffer = make(map[int64][]orderbook.Base) - } - orderbookBuffer[response.ChannelID] = []orderbook.Base{} - if krakenOrderBooks == nil { - krakenOrderBooks = make(map[int64]orderbook.Base) - } - krakenOrderBooks[response.ChannelID] = orderbook.Base{} - return - } - // We change the / to - to maintain compatibility with REST/config pair := currency.NewPairWithDelimiter(response.Pair.Base.String(), response.Pair.Quote.String(), "-") subscriptionChannelPair = append(subscriptionChannelPair, WebsocketChannelData{ @@ -346,16 +321,13 @@ func (k *Kraken) wsProcessOrderBook(channelData *WebsocketChannelData, data inte if asksExist || bidsExist { k.wsRequestMtx.Lock() defer k.wsRequestMtx.Unlock() - k.wsProcessOrderBookBuffer(channelData, obData) - if len(orderbookBuffer[channelData.ChannelID]) >= orderbookBufferLimit { - err := k.wsProcessOrderBookUpdate(channelData) - if err != nil { - subscriptionToRemove := wshandler.WebsocketChannelSubscription{ - Channel: krakenWsOrderbook, - Currency: channelData.Pair, - } - k.Websocket.ResubscribeToChannel(subscriptionToRemove) + err := k.wsProcessOrderBookUpdate(channelData, obData) + if err != nil { + subscriptionToRemove := wshandler.WebsocketChannelSubscription{ + Channel: krakenWsOrderbook, + Currency: channelData.Pair, } + k.Websocket.ResubscribeToChannel(subscriptionToRemove) } } } @@ -363,7 +335,7 @@ func (k *Kraken) wsProcessOrderBook(channelData *WebsocketChannelData, data inte // wsProcessOrderBookPartial creates a new orderbook entry for a given currency pair func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, obData map[string]interface{}) { - ob := orderbook.Base{ + base := orderbook.Base{ Pair: channelData.Pair, AssetType: orderbook.Spot, } @@ -375,11 +347,10 @@ func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, ob asks := askData[i].([]interface{}) price, _ := strconv.ParseFloat(asks[0].(string), 64) amount, _ := strconv.ParseFloat(asks[1].(string), 64) - ob.Asks = append(ob.Asks, orderbook.Item{ + base.Asks = append(base.Asks, orderbook.Item{ Amount: amount, Price: price, }) - timeData, _ := strconv.ParseFloat(asks[2].(string), 64) sec, dec := math.Modf(timeData) askUpdatedTime := time.Unix(int64(sec), int64(dec*(1e9))) @@ -387,17 +358,15 @@ func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, ob highestLastUpdate = askUpdatedTime } } - bidData := obData["bs"].([]interface{}) for i := range bidData { bids := bidData[i].([]interface{}) price, _ := strconv.ParseFloat(bids[0].(string), 64) amount, _ := strconv.ParseFloat(bids[1].(string), 64) - ob.Bids = append(ob.Bids, orderbook.Item{ + base.Bids = append(base.Bids, orderbook.Item{ Amount: amount, Price: price, }) - timeData, _ := strconv.ParseFloat(bids[2].(string), 64) sec, dec := math.Modf(timeData) bidUpdateTime := time.Unix(int64(sec), int64(dec*(1e9))) @@ -405,33 +374,25 @@ func (k *Kraken) wsProcessOrderBookPartial(channelData *WebsocketChannelData, ob highestLastUpdate = bidUpdateTime } } - - ob.LastUpdated = highestLastUpdate - err := k.Websocket.Orderbook.LoadSnapshot(&ob, k.Name, true) + base.LastUpdated = highestLastUpdate + err := k.Websocket.Orderbook.LoadSnapshot(&base, true) if err != nil { k.Websocket.DataHandler <- err return } - k.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: k.Name, Asset: orderbook.Spot, Pair: channelData.Pair, } - - if krakenOrderBooks == nil { - krakenOrderBooks = make(map[int64]orderbook.Base) - } - krakenOrderBooks[channelData.ChannelID] = ob } -func (k *Kraken) wsProcessOrderBookBuffer(channelData *WebsocketChannelData, obData map[string]interface{}) { - ob := orderbook.Base{ - AssetType: orderbook.Spot, - ExchangeName: k.Name, - Pair: channelData.Pair, +// wsProcessOrderBookUpdate updates an orderbook entry for a given currency pair +func (k *Kraken) wsProcessOrderBookUpdate(channelData *WebsocketChannelData, obData map[string]interface{}) error { + update := wsorderbook.WebsocketOrderbookUpdate{ + AssetType: krakenWsAssetType, + CurrencyPair: channelData.Pair, } - var highestLastUpdate time.Time // Ask data is not always sent if _, ok := obData["a"]; ok { @@ -440,11 +401,10 @@ func (k *Kraken) wsProcessOrderBookBuffer(channelData *WebsocketChannelData, obD asks := askData[i].([]interface{}) price, _ := strconv.ParseFloat(asks[0].(string), 64) amount, _ := strconv.ParseFloat(asks[1].(string), 64) - ob.Asks = append(ob.Asks, orderbook.Item{ + update.Asks = append(update.Asks, orderbook.Item{ Amount: amount, Price: price, }) - timeData, _ := strconv.ParseFloat(asks[2].(string), 64) sec, dec := math.Modf(timeData) askUpdatedTime := time.Unix(int64(sec), int64(dec*(1e9))) @@ -460,7 +420,7 @@ func (k *Kraken) wsProcessOrderBookBuffer(channelData *WebsocketChannelData, obD bids := bidData[i].([]interface{}) price, _ := strconv.ParseFloat(bids[0].(string), 64) amount, _ := strconv.ParseFloat(bids[1].(string), 64) - ob.Bids = append(ob.Bids, orderbook.Item{ + update.Bids = append(update.Bids, orderbook.Item{ Amount: amount, Price: price, }) @@ -472,193 +432,20 @@ func (k *Kraken) wsProcessOrderBookBuffer(channelData *WebsocketChannelData, obD } } } - ob.LastUpdated = highestLastUpdate - if orderbookBuffer == nil { - orderbookBuffer = make(map[int64][]orderbook.Base) - } - orderbookBuffer[channelData.ChannelID] = append(orderbookBuffer[channelData.ChannelID], ob) - if k.Verbose { - log.Debugf("%v Adding orderbook to buffer for channel %v. Lastupdated: %v. %v / %v", - k.Name, - channelData.ChannelID, - ob.LastUpdated, - len(orderbookBuffer[channelData.ChannelID]), - orderbookBufferLimit) - } -} - -// wsProcessOrderBookUpdate updates an orderbook entry for a given currency pair -func (k *Kraken) wsProcessOrderBookUpdate(channelData *WebsocketChannelData) error { - if k.Verbose { - log.Debugf("%v Current orderbook 'LastUpdated': %v", - k.Name, - krakenOrderBooks[channelData.ChannelID].LastUpdated) - } - lowestLastUpdated := orderbookBuffer[channelData.ChannelID][0].LastUpdated - if k.Verbose { - log.Debugf("%v Sorting orderbook. Earliest 'LastUpdated' entry: %v", - k.Name, - lowestLastUpdated) - } - sort.Slice(orderbookBuffer[channelData.ChannelID], func(i, j int) bool { - return orderbookBuffer[channelData.ChannelID][i].LastUpdated.Before(orderbookBuffer[channelData.ChannelID][j].LastUpdated) - }) - - lowestLastUpdated = orderbookBuffer[channelData.ChannelID][0].LastUpdated - if k.Verbose { - log.Debugf("%v Sorted orderbook. Earliest 'LastUpdated' entry: %v", - k.Name, - lowestLastUpdated) - } - // The earliest update has to be after the previously stored orderbook - if krakenOrderBooks[channelData.ChannelID].LastUpdated.After(lowestLastUpdated) { - err := fmt.Errorf("%v orderbook update out of order. Existing: %v, Attempted: %v", - k.Name, - krakenOrderBooks[channelData.ChannelID].LastUpdated, - lowestLastUpdated) - k.Websocket.DataHandler <- err - return err - } - - k.updateChannelOrderbookEntries(channelData) - highestLastUpdate := orderbookBuffer[channelData.ChannelID][len(orderbookBuffer[channelData.ChannelID])-1].LastUpdated - if k.Verbose { - log.Debugf("%v Saving orderbook. Lastupdated: %v", - k.Name, - highestLastUpdate) - } - - ob := krakenOrderBooks[channelData.ChannelID] - ob.LastUpdated = highestLastUpdate - err := k.Websocket.Orderbook.LoadSnapshot(&ob, k.Name, true) + update.UpdateTime = highestLastUpdate + err := k.Websocket.Orderbook.Update(&update) if err != nil { k.Websocket.DataHandler <- err return err } - k.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: k.Name, Asset: orderbook.Spot, Pair: channelData.Pair, } - // Reset the buffer - orderbookBuffer[channelData.ChannelID] = []orderbook.Base{} return nil } -func (k *Kraken) updateChannelOrderbookEntries(channelData *WebsocketChannelData) { - for i := 0; i < len(orderbookBuffer[channelData.ChannelID]); i++ { - for j := 0; j < len(orderbookBuffer[channelData.ChannelID][i].Asks); j++ { - k.updateChannelOrderbookAsks(i, j, channelData) - } - for j := 0; j < len(orderbookBuffer[channelData.ChannelID][i].Bids); j++ { - k.updateChannelOrderbookBids(i, j, channelData) - } - } -} - -func (k *Kraken) updateChannelOrderbookAsks(i, j int, channelData *WebsocketChannelData) { - askFound := k.updateChannelOrderbookAsk(i, j, channelData) - if !askFound { - if k.Verbose { - log.Debugf("%v Adding Ask for channel %v. Price %v. Amount %v", - k.Name, - channelData.ChannelID, - orderbookBuffer[channelData.ChannelID][i].Asks[j].Price, - orderbookBuffer[channelData.ChannelID][i].Asks[j].Amount) - } - ob := krakenOrderBooks[channelData.ChannelID] - ob.Asks = append(ob.Asks, orderbookBuffer[channelData.ChannelID][i].Asks[j]) - krakenOrderBooks[channelData.ChannelID] = ob - } -} - -func (k *Kraken) updateChannelOrderbookAsk(i, j int, channelData *WebsocketChannelData) bool { - askFound := false - for l := 0; l < len(krakenOrderBooks[channelData.ChannelID].Asks); l++ { - if krakenOrderBooks[channelData.ChannelID].Asks[l].Price == orderbookBuffer[channelData.ChannelID][i].Asks[j].Price { - askFound = true - if orderbookBuffer[channelData.ChannelID][i].Asks[j].Amount == 0 { - // Remove existing entry - if k.Verbose { - log.Debugf("%v Removing Ask for channel %v. Price %v. Old amount %v. Buffer %v", - k.Name, - channelData.ChannelID, - orderbookBuffer[channelData.ChannelID][i].Asks[j].Price, - krakenOrderBooks[channelData.ChannelID].Asks[l].Amount, i) - } - ob := krakenOrderBooks[channelData.ChannelID] - ob.Asks = append(ob.Asks[:l], ob.Asks[l+1:]...) - krakenOrderBooks[channelData.ChannelID] = ob - l-- - } else if krakenOrderBooks[channelData.ChannelID].Asks[l].Amount != orderbookBuffer[channelData.ChannelID][i].Asks[j].Amount { - if k.Verbose { - log.Debugf("%v Updating Ask for channel %v. Price %v. Old amount %v, New Amount %v", - k.Name, - channelData.ChannelID, - orderbookBuffer[channelData.ChannelID][i].Asks[j].Price, - krakenOrderBooks[channelData.ChannelID].Asks[l].Amount, - orderbookBuffer[channelData.ChannelID][i].Asks[j].Amount) - } - krakenOrderBooks[channelData.ChannelID].Asks[l].Amount = orderbookBuffer[channelData.ChannelID][i].Asks[j].Amount - } - return askFound - } - } - return askFound -} - -func (k *Kraken) updateChannelOrderbookBids(i, j int, channelData *WebsocketChannelData) { - bidFound := k.updateChannelOrderbookBid(i, j, channelData) - if !bidFound { - if k.Verbose { - log.Debugf("%v Adding Bid for channel %v. Price %v. Amount %v", - k.Name, - channelData.ChannelID, - orderbookBuffer[channelData.ChannelID][i].Bids[j].Price, - orderbookBuffer[channelData.ChannelID][i].Bids[j].Amount) - } - ob := krakenOrderBooks[channelData.ChannelID] - ob.Bids = append(ob.Bids, orderbookBuffer[channelData.ChannelID][i].Bids[j]) - krakenOrderBooks[channelData.ChannelID] = ob - } -} - -func (k *Kraken) updateChannelOrderbookBid(i, j int, channelData *WebsocketChannelData) bool { - bidFound := false - for l := 0; l < len(krakenOrderBooks[channelData.ChannelID].Bids); l++ { - if krakenOrderBooks[channelData.ChannelID].Bids[l].Price == orderbookBuffer[channelData.ChannelID][i].Bids[j].Price { - bidFound = true - if orderbookBuffer[channelData.ChannelID][i].Bids[j].Amount == 0 { - // Remove existing entry - if k.Verbose { - log.Debugf("%v Removing Bid for channel %v. Price %v. Old amount %v. Buffer %v", - k.Name, - channelData.ChannelID, - orderbookBuffer[channelData.ChannelID][i].Bids[j].Price, - krakenOrderBooks[channelData.ChannelID].Bids[l].Amount, i) - } - ob := krakenOrderBooks[channelData.ChannelID] - ob.Bids = append(ob.Bids[:l], ob.Bids[l+1:]...) - krakenOrderBooks[channelData.ChannelID] = ob - l-- - } else if krakenOrderBooks[channelData.ChannelID].Bids[l].Amount != orderbookBuffer[channelData.ChannelID][i].Bids[j].Amount { - if k.Verbose { - log.Debugf("%v Updating Bid for channel %v. Price %v. Old amount %v, New Amount %v", - k.Name, - channelData.ChannelID, - orderbookBuffer[channelData.ChannelID][i].Bids[j].Price, - krakenOrderBooks[channelData.ChannelID].Bids[l].Amount, - orderbookBuffer[channelData.ChannelID][i].Bids[j].Amount) - } - krakenOrderBooks[channelData.ChannelID].Bids[l].Amount = orderbookBuffer[channelData.ChannelID][i].Bids[j].Amount - } - return bidFound - } - } - return bidFound -} - // wsProcessCandles converts candle data and sends it to the data handler func (k *Kraken) wsProcessCandles(channelData *WebsocketChannelData, data interface{}) { candleData := data.([]interface{}) diff --git a/exchanges/kraken/kraken_wrapper.go b/exchanges/kraken/kraken_wrapper.go index 83b74d1a..2177f1a2 100644 --- a/exchanges/kraken/kraken_wrapper.go +++ b/exchanges/kraken/kraken_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/lakebtc/lakebtc.go b/exchanges/lakebtc/lakebtc.go index 79cfa7d9..afc61cee 100644 --- a/exchanges/lakebtc/lakebtc.go +++ b/exchanges/lakebtc/lakebtc.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -41,6 +41,7 @@ const ( // LakeBTC is the overarching type across the LakeBTC package type LakeBTC struct { exchange.Base + WebsocketConn } // SetDefaults sets LakeBTC defaults @@ -67,6 +68,10 @@ func (l *LakeBTC) SetDefaults() { l.APIUrlDefault = lakeBTCAPIURL l.APIUrl = l.APIUrlDefault l.Websocket = wshandler.New() + l.Websocket.Functionality = wshandler.WebsocketOrderbookSupported | + wshandler.WebsocketTradeDataSupported | + wshandler.WebsocketSubscribeSupported + l.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets exchange configuration profile @@ -105,6 +110,25 @@ func (l *LakeBTC) Setup(exch *config.ExchangeConfig) { if err != nil { log.Fatal(err) } + err = l.Websocket.Setup(l.WsConnect, + l.Subscribe, + nil, + exch.Name, + exch.Websocket, + exch.Verbose, + lakeBTCWSURL, + exch.WebsocketURL, + exch.AuthenticatedWebsocketAPISupport) + if err != nil { + log.Fatal(err) + } + l.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + false, + false, + false, + false, + exch.Name) } } diff --git a/exchanges/lakebtc/lakebtc_test.go b/exchanges/lakebtc/lakebtc_test.go index 7a593cd1..2903b6a4 100644 --- a/exchanges/lakebtc/lakebtc_test.go +++ b/exchanges/lakebtc/lakebtc_test.go @@ -1,15 +1,19 @@ package lakebtc import ( + "fmt" "testing" "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" + "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) var l LakeBTC +var setupRan bool // Please add your own APIkeys to do correct due diligence testing. const ( @@ -19,21 +23,27 @@ const ( ) func TestSetDefaults(t *testing.T) { - l.SetDefaults() + if !setupRan { + l.SetDefaults() + } } func TestSetup(t *testing.T) { - cfg := config.GetConfig() - cfg.LoadConfig("../../testdata/configtest.json") - lakebtcConfig, err := cfg.GetExchangeConfig("LakeBTC") - if err != nil { - t.Error("Test Failed - LakeBTC Setup() init error") + if !setupRan { + cfg := config.GetConfig() + cfg.LoadConfig("../../testdata/configtest.json") + lakebtcConfig, err := cfg.GetExchangeConfig("LakeBTC") + if err != nil { + t.Error("Test Failed - LakeBTC Setup() init error") + } + lakebtcConfig.AuthenticatedAPISupport = true + lakebtcConfig.APIKey = apiKey + lakebtcConfig.APISecret = apiSecret + lakebtcConfig.Websocket = true + l.Setup(&lakebtcConfig) + l.WebsocketURL = lakeBTCWSURL + setupRan = true } - lakebtcConfig.AuthenticatedAPISupport = true - lakebtcConfig.APIKey = apiKey - lakebtcConfig.APISecret = apiSecret - - l.Setup(&lakebtcConfig) } func TestGetTradablePairs(t *testing.T) { @@ -445,3 +455,65 @@ func TestGetDepositAddress(t *testing.T) { } } } + +// TestWsConn websocket connection test +func TestWsConn(t *testing.T) { + TestSetDefaults(t) + TestSetup(t) + if !l.Websocket.IsEnabled() { + t.Skip(wshandler.WebsocketNotEnabled) + } + l.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride() + l.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride() + err := l.WsConnect() + if err != nil { + t.Fatal(err) + } +} + +// TestWsTradeProcessing logic test +func TestWsTradeProcessing(t *testing.T) { + TestSetDefaults(t) + TestSetup(t) + l.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride() + l.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride() + json := `{"trades":[{"type":"sell","date":1564985787,"price":"11913.02","amount":"0.49"}]}` + err := l.processTrades(json, "market-btcusd-global") + if err != nil { + t.Error(err) + } +} + +// TestWsTickerProcessing logic test +func TestWsTickerProcessing(t *testing.T) { + TestSetDefaults(t) + TestSetup(t) + l.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride() + l.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride() + json := `{"btcusd":{"low":"10990.05","high":"11966.24","last":"11903.29","volume":"1803.967079","sell":"11912.39","buy":"11902.2"},"btceur":{"low":"9886.87","high":"10732.72","last":"10691.44","volume":"87.994478","sell":"10711.62","buy":"10691.44"},"btchkd":{"low":null,"high":null,"last":"51776.98","volume":null,"sell":"93307.37","buy":"93177.56"},"btcjpy":{"low":"1176039.0","high":"1272246.0","last":"1265680.0","volume":"129.021421","sell":"1266764.0","buy":"1265680.0"},"btcgbp":{"low":"9157.12","high":"9953.43","last":"9941.28","volume":"10.4997","sell":"10007.89","buy":"9941.28"},"btcaud":{"low":"16102.57","high":"17594.22","last":"17548.16","volume":"7.338316","sell":"17616.67","buy":"17549.69"},"btccad":{"low":"14541.69","high":"15834.87","last":"15763.54","volume":"30.480309","sell":"15793.45","buy":"15756.13"},"btcsgd":{"low":"15133.82","high":"16501.62","last":"16455.53","volume":"4.044026","sell":"16484.37","buy":"16462.18"},"btcchf":{"low":"10800.58","high":"11526.24","last":"11526.24","volume":"0.1765","sell":"11675.34","buy":"11632.02"},"btcnzd":{"low":null,"high":null,"last":"8340.98","volume":null,"sell":"18315.49","buy":"18221.37"},"btcngn":{"low":null,"high":null,"last":"600000.0","volume":null,"sell":null,"buy":null},"eurusd":{"low":"1.1088","high":"1.1138","last":"1.1125","volume":"2680.105249","sell":"1.1142","buy":"1.1121"},"gbpusd":{"low":"1.1934","high":"1.1958","last":"1.1934","volume":"1493.923823","sell":"1.1979","buy":"1.1903"},"usdjpy":{"low":"105.26","high":"107.25","last":"106.33","volume":"114490.2179","sell":"106.34","buy":"106.27"},"usdhkd":{"low":null,"high":null,"last":"7.851","volume":null,"sell":"7.8328","buy":"7.8286"},"usdcad":{"low":"1.3225","high":"1.3272","last":"1.3255","volume":"11033.9877","sell":"1.3258","buy":"1.3238"},"usdsgd":{"low":"1.3776","high":"1.3839","last":"1.3838","volume":"2523.75","sell":"1.3838","buy":"1.3819"},"audusd":{"low":"0.6764","high":"0.6853","last":"0.6771","volume":"5442.608321","sell":"0.6782","buy":"0.6762"},"nzdusd":{"low":null,"high":null,"last":"0.6758","volume":null,"sell":"0.6532","buy":"0.6504"},"usdchf":{"low":"0.9838","high":"0.9838","last":"0.9838","volume":"108.3352","sell":"0.9801","buy":"0.9773"},"usdngn":{"low":null,"high":null,"last":"200.0","volume":null,"sell":null,"buy":null},"ethbtc":{"low":"0.0205","high":"0.025","last":"0.0205","volume":null,"sell":"0.03","buy":"0.0194"},"ltcbtc":{"low":null,"high":null,"last":"0.0114","volume":null,"sell":"0.009","buy":"0.0073"},"bchbtc":{"low":null,"high":null,"last":"0.0544","volume":null,"sell":"0.0322","buy":"0.0274"},"xrpbtc":{"low":"0.000042","high":"0.000042","last":"0.000042","volume":null,"sell":"0.000037","buy":"0.000022"},"baceth":{"low":"0.000035","high":"0.000035","last":"0.000035","volume":null,"sell":"0.0015","buy":null}}` + err := l.processTicker(json) + if err != nil { + t.Error(err) + } +} + +func TestGetCurrencyFromChannel(t *testing.T) { + curr := currency.NewPair(currency.LTC, currency.BTC) + result := l.getCurrencyFromChannel(fmt.Sprintf("%v%v%v", marketSubstring, curr, globalSubstring)) + if curr != result { + t.Errorf("currency result is not equal. Expected %v", curr) + } +} + +// TestWsOrderbookProcessing logic test +func TestWsOrderbookProcessing(t *testing.T) { + TestSetDefaults(t) + TestSetup(t) + l.Websocket.DataHandler = sharedtestvalues.GetWebsocketInterfaceChannelOverride() + l.Websocket.TrafficAlert = sharedtestvalues.GetWebsocketStructChannelOverride() + json := `{"asks":[["11905.66","0.0019"],["11905.73","0.0015"],["11906.43","0.0013"],["11906.62","0.0019"],["11907.25","11.087"],["11907.66","0.0006"],["11907.73","0.3113"],["11907.84","0.0006"],["11908.37","0.0016"],["11908.86","10.3786"],["11909.54","4.2955"],["11910.15","0.0012"],["11910.56","13.5505"],["11911.06","0.0011"],["11911.37","0.0023"]],"bids":[["11905.55","0.0171"],["11904.43","0.0225"],["11903.31","0.0223"],["11902.2","0.0027"],["11901.92","1.002"],["11901.6","0.0015"],["11901.49","0.0012"],["11901.08","0.0227"],["11900.93","0.0009"],["11900.53","1.662"],["11900.08","0.001"],["11900.01","3.6745"],["11899.96","0.003"],["11899.91","0.0006"],["11899.44","0.0013"]]}` + err := l.processOrderbook(json, "market-btcusd-global") + if err != nil { + t.Error(err) + } +} diff --git a/exchanges/lakebtc/lakebtc_types.go b/exchanges/lakebtc/lakebtc_types.go index 7bdda80e..7fd6abff 100644 --- a/exchanges/lakebtc/lakebtc_types.go +++ b/exchanges/lakebtc/lakebtc_types.go @@ -1,5 +1,7 @@ package lakebtc +import pusher "github.com/toorop/go-pusher" + // Ticker holds ticker information type Ticker struct { Last float64 @@ -112,3 +114,30 @@ type Withdraw struct { At int64 `json:"at"` Error string `json:"error"` } + +// WebsocketConn defines a pusher websocket connection +type WebsocketConn struct { + Client *pusher.Client + Ticker chan *pusher.Event + Orderbook chan *pusher.Event + Trade chan *pusher.Event +} + +// WsOrderbookUpdate contains orderbook data from websocket +type WsOrderbookUpdate struct { + Asks [][]string `json:"asks"` + Bids [][]string `json:"bids"` +} + +// WsTrades contains trade data from websocket +type WsTrades struct { + Trades []WsTrade `json:"trades"` +} + +// WsTrade contains individual trade details from websocket +type WsTrade struct { + Type string `json:"type"` + Date int64 `json:"date"` + Price float64 `json:"price,string"` + Amount float64 `json:"amount,string"` +} diff --git a/exchanges/lakebtc/lakebtc_websocket.go b/exchanges/lakebtc/lakebtc_websocket.go new file mode 100644 index 00000000..1cdac6a2 --- /dev/null +++ b/exchanges/lakebtc/lakebtc_websocket.go @@ -0,0 +1,248 @@ +package lakebtc + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/thrasher-corp/gocryptotrader/common" + "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + log "github.com/thrasher-corp/gocryptotrader/logger" + "github.com/toorop/go-pusher" +) + +const ( + lakeBTCWSURL = "ws.lakebtc.com:8085" + marketGlobalEndpoint = "market-global" + marketSubstring = "market-" + globalSubstring = "-global" + volumeString = "volume" + highString = "high" + lowString = "low" + wssSchem = "wss" +) + +// WsConnect initiates a new websocket connection +func (l *LakeBTC) WsConnect() error { + if !l.Websocket.IsEnabled() || !l.IsEnabled() { + return errors.New(wshandler.WebsocketNotEnabled) + } + var err error + l.WebsocketConn.Client, err = pusher.NewCustomClient(strings.ToLower(l.Name), lakeBTCWSURL, wssSchem) + if err != nil { + return err + } + err = l.WebsocketConn.Client.Subscribe(marketGlobalEndpoint) + if err != nil { + return err + } + l.GenerateDefaultSubscriptions() + err = l.listenToEndpoints() + if err != nil { + return err + } + go l.wsHandleIncomingData() + return nil +} + +func (l *LakeBTC) listenToEndpoints() error { + var err error + l.WebsocketConn.Ticker, err = l.WebsocketConn.Client.Bind("tickers") + if err != nil { + return fmt.Errorf("%s Websocket Bind error: %s", l.GetName(), err) + } + l.WebsocketConn.Orderbook, err = l.WebsocketConn.Client.Bind("update") + if err != nil { + return fmt.Errorf("%s Websocket Bind error: %s", l.GetName(), err) + } + l.WebsocketConn.Trade, err = l.WebsocketConn.Client.Bind("trades") + if err != nil { + return fmt.Errorf("%s Websocket Bind error: %s", l.GetName(), err) + } + return nil +} + +// GenerateDefaultSubscriptions Adds default subscriptions to websocket to be handled by ManageSubscriptions() +func (l *LakeBTC) GenerateDefaultSubscriptions() { + var subscriptions []wshandler.WebsocketChannelSubscription + enabledCurrencies := l.GetEnabledCurrencies() + for j := range enabledCurrencies { + enabledCurrencies[j].Delimiter = "" + channel := fmt.Sprintf("%v%v%v", marketSubstring, enabledCurrencies[j].Lower(), globalSubstring) + subscriptions = append(subscriptions, wshandler.WebsocketChannelSubscription{ + Channel: channel, + Currency: enabledCurrencies[j], + }) + } + l.Websocket.SubscribeToChannels(subscriptions) +} + +// Subscribe sends a websocket message to receive data from the channel +func (l *LakeBTC) Subscribe(channelToSubscribe wshandler.WebsocketChannelSubscription) error { + return l.WebsocketConn.Client.Subscribe(channelToSubscribe.Channel) +} + +// wsHandleIncomingData services incoming data from the websocket connection +func (l *LakeBTC) wsHandleIncomingData() { + l.Websocket.Wg.Add(1) + defer l.Websocket.Wg.Done() + for { + select { + case <-l.Websocket.ShutdownC: + return + case data := <-l.WebsocketConn.Ticker: + if l.Verbose { + log.Debugf("%v Websocket message received: %v", l.Name, data) + } + l.Websocket.TrafficAlert <- struct{}{} + err := l.processTicker(data.Data) + if err != nil { + l.Websocket.DataHandler <- err + return + } + case data := <-l.WebsocketConn.Trade: + l.Websocket.TrafficAlert <- struct{}{} + if l.Verbose { + log.Debugf("%v Websocket message received: %v", l.Name, data) + } + err := l.processTrades(data.Data, data.Channel) + if err != nil { + l.Websocket.DataHandler <- err + return + } + case data := <-l.WebsocketConn.Orderbook: + l.Websocket.TrafficAlert <- struct{}{} + if l.Verbose { + log.Debugf("%v Websocket message received: %v", l.Name, data) + } + err := l.processOrderbook(data.Data, data.Channel) + if err != nil { + l.Websocket.DataHandler <- err + return + } + } + } +} + +func (l *LakeBTC) processTrades(data, channel string) error { + var tradeData WsTrades + err := common.JSONDecode([]byte(data), &tradeData) + if err != nil { + return err + } + curr := l.getCurrencyFromChannel(channel) + for i := 0; i < len(tradeData.Trades); i++ { + l.Websocket.DataHandler <- wshandler.TradeData{ + Timestamp: time.Unix(tradeData.Trades[i].Date, 0), + CurrencyPair: curr, + AssetType: orderbook.Spot, + Exchange: l.GetName(), + EventType: orderbook.Spot, + EventTime: tradeData.Trades[i].Date, + Price: tradeData.Trades[i].Price, + Amount: tradeData.Trades[i].Amount, + Side: tradeData.Trades[i].Type, + } + } + return nil +} + +func (l *LakeBTC) processOrderbook(obUpdate, channel string) error { + var update WsOrderbookUpdate + err := common.JSONDecode([]byte(obUpdate), &update) + if err != nil { + return err + } + book := orderbook.Base{ + Pair: l.getCurrencyFromChannel(channel), + LastUpdated: time.Now(), + AssetType: orderbook.Spot, + ExchangeName: l.Name, + } + + for i := 0; i < len(update.Asks); i++ { + var amount, price float64 + amount, err = strconv.ParseFloat(update.Asks[i][1], 64) + if err != nil { + l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'low' %v", l.Name, update.Asks[i]) + continue + } + price, err = strconv.ParseFloat(update.Asks[i][0], 64) + if err != nil { + l.Websocket.DataHandler <- fmt.Errorf("%v error parsing orderbook price %v", l.Name, update.Asks[i]) + continue + } + book.Asks = append(book.Asks, orderbook.Item{ + Amount: amount, + Price: price, + }) + } + for i := 0; i < len(update.Bids); i++ { + var amount, price float64 + amount, err = strconv.ParseFloat(update.Bids[i][1], 64) + if err != nil { + l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'low' %v", l.Name, update.Bids[i]) + continue + } + price, err = strconv.ParseFloat(update.Bids[i][0], 64) + if err != nil { + l.Websocket.DataHandler <- fmt.Errorf("%v error parsing orderbook price %v", l.Name, update.Bids[i]) + continue + } + book.Bids = append(book.Bids, orderbook.Item{ + Amount: amount, + Price: price, + }) + } + return l.Websocket.Orderbook.LoadSnapshot(&book, true) +} + +func (l *LakeBTC) getCurrencyFromChannel(channel string) currency.Pair { + curr := strings.Replace(channel, marketSubstring, "", 1) + curr = strings.Replace(curr, globalSubstring, "", 1) + return currency.NewPairFromString(curr) +} + +func (l *LakeBTC) processTicker(ticker string) error { + var tUpdate map[string]interface{} + err := common.JSONDecode([]byte(ticker), &tUpdate) + if err != nil { + l.Websocket.DataHandler <- err + return err + } + for k, v := range tUpdate { + tickerData := v.(map[string]interface{}) + if tickerData[highString] == nil || tickerData[lowString] == nil || tickerData[volumeString] == nil { + continue + } + high, err := strconv.ParseFloat(tickerData[highString].(string), 64) + if err != nil { + l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'high' %v", l.Name, tickerData) + continue + } + low, err := strconv.ParseFloat(tickerData[lowString].(string), 64) + if err != nil { + l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'low' %v", l.Name, tickerData) + continue + } + vol, err := strconv.ParseFloat(tickerData[volumeString].(string), 64) + if err != nil { + l.Websocket.DataHandler <- fmt.Errorf("%v error parsing ticker data 'volume' %v", l.Name, tickerData) + continue + } + l.Websocket.DataHandler <- wshandler.TickerData{ + Timestamp: time.Now(), + Pair: currency.NewPairFromString(k), + AssetType: orderbook.Spot, + Exchange: l.GetName(), + Quantity: vol, + HighPrice: high, + LowPrice: low, + } + } + return nil +} diff --git a/exchanges/lakebtc/lakebtc_wrapper.go b/exchanges/lakebtc/lakebtc_wrapper.go index 0cd973e8..d898a32a 100644 --- a/exchanges/lakebtc/lakebtc_wrapper.go +++ b/exchanges/lakebtc/lakebtc_wrapper.go @@ -13,7 +13,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -268,8 +268,7 @@ func (l *LakeBTC) WithdrawFiatFundsToInternationalBank(withdrawRequest *exchange // GetWebsocket returns a pointer to the exchange websocket func (l *LakeBTC) GetWebsocket() (*wshandler.Websocket, error) { - // Documents are too vague to implement - return nil, common.ErrFunctionNotSupported + return l.Websocket, nil } // GetFeeByType returns an estimate of fee based on type of transaction diff --git a/exchanges/localbitcoins/localbitcoins.go b/exchanges/localbitcoins/localbitcoins.go index 135eafee..e9d0a581 100644 --- a/exchanges/localbitcoins/localbitcoins.go +++ b/exchanges/localbitcoins/localbitcoins.go @@ -13,7 +13,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/config" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/localbitcoins/localbitcoins_wrapper.go b/exchanges/localbitcoins/localbitcoins_wrapper.go index 726226c6..f065bc0a 100644 --- a/exchanges/localbitcoins/localbitcoins_wrapper.go +++ b/exchanges/localbitcoins/localbitcoins_wrapper.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/okcoin/okcoin.go b/exchanges/okcoin/okcoin.go index 43402d14..896b3496 100644 --- a/exchanges/okcoin/okcoin.go +++ b/exchanges/okcoin/okcoin.go @@ -8,7 +8,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/exchanges/okgroup" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) const ( @@ -62,4 +62,5 @@ func (o *OKCoin) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported o.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit o.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + o.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } diff --git a/exchanges/okcoin/okcoin_test.go b/exchanges/okcoin/okcoin_test.go index 8835814e..b22a6d2e 100644 --- a/exchanges/okcoin/okcoin_test.go +++ b/exchanges/okcoin/okcoin_test.go @@ -13,8 +13,9 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/okgroup" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please supply you own test keys here for due diligence testing. @@ -849,7 +850,7 @@ func TestSendWsMessages(t *testing.T) { func TestGetAssetTypeFromTableName(t *testing.T) { str := "spot/candle300s:BTC-USDT" spot := o.GetAssetTypeFromTableName(str) - if spot != "SPOT" { + if spot != orderbook.Spot { t.Errorf("Error, expected 'SPOT', received: '%v'", spot) } } diff --git a/exchanges/okex/okex.go b/exchanges/okex/okex.go index 74678e58..7382459a 100644 --- a/exchanges/okex/okex.go +++ b/exchanges/okex/okex.go @@ -11,7 +11,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/exchanges/okgroup" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) const ( @@ -87,6 +87,7 @@ func (o *OKEX) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported o.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit o.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + o.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } diff --git a/exchanges/okex/okex_test.go b/exchanges/okex/okex_test.go index 3a1042e2..481d8fb8 100644 --- a/exchanges/okex/okex_test.go +++ b/exchanges/okex/okex_test.go @@ -14,8 +14,9 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/okgroup" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please supply you own test keys here for due diligence testing. @@ -1614,7 +1615,7 @@ func TestSendWsMessages(t *testing.T) { func TestGetAssetTypeFromTableName(t *testing.T) { str := "spot/candle300s:BTC-USDT" spot := o.GetAssetTypeFromTableName(str) - if spot != "SPOT" { + if spot != orderbook.Spot { t.Errorf("Error, expected 'SPOT', received: '%v'", spot) } } @@ -1645,9 +1646,6 @@ func TestGetWsChannelWithoutOrderType(t *testing.T) { // TestOrderBookUpdateChecksumCalculator logic test func TestOrderBookUpdateChecksumCalculator(t *testing.T) { TestSetDefaults(t) - if !websocketEnabled { - t.Skip("Websocket not enabled, skipping") - } original := `{"table":"spot/depth","action":"partial","data":[{"instrument_id":"BTC-USDT","asks":[["3864.6786","0.145",1],["3864.7682","0.005",1],["3864.9851","0.57",1],["3864.9852","0.30137754",1],["3864.9986","2.81818419",1],["3864.9995","0.002",1],["3865","0.0597",1],["3865.0309","0.4",1],["3865.1995","0.004",1],["3865.3995","0.004",1],["3865.5995","0.004",1],["3865.7995","0.004",1],["3865.9995","0.004",1],["3866.0961","0.25865886",1],["3866.1995","0.004",1],["3866.3995","0.004",1],["3866.4004","0.3243",2],["3866.5995","0.004",1],["3866.7633","0.44247086",1],["3866.7995","0.004",1],["3866.9197","0.511",1],["3867.256","0.51716256",1],["3867.3951","0.02588112",1],["3867.4014","0.025",1],["3867.4566","0.02499999",1],["3867.4675","4.01155057",5],["3867.5515","1.1",1],["3867.6113","0.009",1],["3867.7349","0.026",1],["3867.7781","0.03738652",1],["3867.9163","0.0521",1],["3868.0381","0.34354941",1],["3868.0436","0.051",1],["3868.0657","0.90552172",3],["3868.1819","0.03863346",1],["3868.2013","0.194",1],["3868.346","0.051",1],["3868.3863","0.01155",1],["3868.7716","0.009",1],["3868.947","0.025",1],["3868.98","0.001",1],["3869.0764","1.03487931",1],["3869.2773","0.07724578",1],["3869.4039","0.025",1],["3869.4068","1.03",1],["3869.7068","2.06976398",1],["3870","0.5",1],["3870.0465","0.01",1],["3870.7042","0.02099651",1],["3870.9451","2.07047375",1],["3871.5254","1.2",1],["3871.5596","0.001",1],["3871.6605","0.01035032",1],["3871.7179","2.07047375",1],["3871.8816","0.51751625",1],["3872.1","0.75",1],["3872.2464","0.0646",1],["3872.3747","0.283",1],["3872.4039","0.2",1],["3872.7655","0.23179307",1],["3872.8005","2.06976398",1],["3873.1509","2",1],["3873.3215","0.26",1],["3874.1392","0.001",1],["3874.1487","3.88224364",4],["3874.1685","1.8",1],["3874.5571","0.08974762",1],["3874.734","2.06976398",1],["3874.99","0.3",1],["3875","1.001",2],["3875.0041","1.03505051",1],["3875.45","0.3",1],["3875.4766","0.15",1],["3875.7057","0.51751625",1],["3876","0.001",1],["3876.68","0.3",1],["3876.7188","0.001",1],["3877","0.75",1],["3877.31","0.035",1],["3877.38","0.3",1],["3877.7","0.3",1],["3877.88","0.3",1],["3878.0364","0.34770122",1],["3878.4525","0.48579748",1],["3878.4955","0.02812511",1],["3878.8855","0.00258579",1],["3878.9605","0.895",1],["3879","0.001",1],["3879.2984","0.002",2],["3879.432","0.001",1],["3879.6313","6",1],["3879.9999","0.002",2],["3880","1.25132834",5],["3880.2526","0.04075162",1],["3880.7145","0.0647",1],["3881.2469","1.883",1],["3881.878","0.002",2],["3884.4576","0.002",2],["3885","0.002",2],["3885.2233","0.28304103",1],["3885.7416","18",1],["3886","0.001",1],["3886.1554","5.4",1],["3887","0.001",1],["3887.0372","0.002",2],["3887.2559","0.05214011",1],["3887.9238","0.0019",1],["3888","0.15810538",4],["3889","0.001",1],["3889.5175","0.50510653",1],["3889.6168","0.002",2],["3889.9999","0.001",1],["3890","2.34968109",4],["3890.5222","0.00257806",1],["3891.2659","5",1],["3891.9999","0.00893897",1],["3892.1964","0.002",2],["3892.4358","0.0176",1],["3893.1388","1.4279",1],["3894","0.0026321",1],["3894.776","0.001",1],["3895","1.501",2],["3895.379","0.25881288",1],["3897","0.05",1],["3897.3556","0.001",1],["3897.8432","0.73708079",1],["3898","3.31353018",7],["3898.4462","4.757",1],["3898.6","0.47159638",1],["3898.8769","0.0129",1],["3899","6",2],["3899.6516","0.025",1],["3899.9352","0.001",1],["3899.9999","0.013",2],["3900","22.37447743",24],["3900.9999","0.07763916",1],["3901","0.10192487",1],["3902.1937","0.00257034",1],["3902.3991","1.5532141",1],["3902.5148","0.001",1],["3904","1.49331984",1],["3904.9999","0.95905447",1],["3905","0.501",2],["3905.0944","0.001",1],["3905.61","0.099",1],["3905.6801","0.54343686",1],["3906.2901","0.0258",1],["3907.674","0.001",1],["3907.85","1.35778084",1],["3908","0.03846153",1],["3908.23","1.95189531",1],["3908.906","0.03148978",1],["3909","0.001",1],["3909.9999","0.01398721",2],["3910","0.016",2],["3910.2536","0.001",1],["3912.5406","0.88270517",1],["3912.8332","0.001",1],["3913","1.2640608",1],["3913.87","1.69114184",1],["3913.9003","0.00256266",1],["3914","1.21766411",1],["3915","0.001",1],["3915.4128","0.001",1],["3915.7425","6.848",1],["3916","0.0050949",1],["3917.36","1.28658296",1],["3917.9924","0.001",1],["3919","0.001",1],["3919.9999","0.001",1],["3920","1.21171832",3],["3920.0002","0.20217038",1],["3920.572","0.001",1],["3921","0.128",1],["3923.0756","0.00148064",1],["3923.1516","0.001",1],["3923.86","1.38831714",1],["3925","0.01867801",2],["3925.642","0.00255499",1],["3925.7312","0.001",1],["3926","0.04290757",1],["3927","0.023",1],["3927.3175","0.01212865",1],["3927.65","1.51375612",1],["3928","0.5",1],["3928.3108","0.001",1],["3929","0.001",1],["3929.9999","0.01519338",2],["3930","0.0174985",3],["3930.21","1.49335799",1],["3930.8904","0.001",1],["3932.2999","0.01953",1],["3932.8962","7.96",1],["3933.0387","11.808",1],["3933.47","0.001",1],["3934","1.40839932",1],["3935","0.001",1],["3936.8","0.62879518",1],["3937.23","1.56977841",1],["3937.4189","0.00254735",1]],"bids":[["3864.5217","0.00540709",1],["3864.5216","0.14068758",2],["3864.2275","0.01033576",1],["3864.0989","0.00825047",1],["3864.0273","0.38",1],["3864.0272","0.4",1],["3863.9957","0.01083539",1],["3863.9184","0.01653723",1],["3863.8282","0.25588165",1],["3863.8153","0.154",1],["3863.7791","1.14122492",1],["3863.6866","0.01733662",1],["3863.6093","0.02645958",1],["3863.3775","0.02773862",1],["3863.0297","0.513",1],["3863.0286","1.1028564",2],["3862.8489","0.01",1],["3862.5972","0.01890179",1],["3862.3431","0.01152944",1],["3862.313","0.009",1],["3862.2445","0.90551002",3],["3862.0734","0.014",1],["3862.0539","0.64976067",1],["3861.8586","0.025",1],["3861.7888","0.025",1],["3861.7673","0.008",1],["3861.5785","0.01",1],["3861.3895","0.005",1],["3861.3338","0.25875855",1],["3861.161","0.01",1],["3861.1111","0.03863352",1],["3861.0732","0.51703882",1],["3860.9116","0.17754895",1],["3860.75","0.19",1],["3860.6554","0.015",1],["3860.6172","0.005",1],["3860.6088","0.008",1],["3860.4724","0.12940042",1],["3860.4424","0.25880084",1],["3860.42","0.01",1],["3860.3725","0.51760102",1],["3859.8449","0.005",1],["3859.8285","0.03738652",1],["3859.7638","0.07726703",1],["3859.4502","0.008",1],["3859.3772","0.05173471",1],["3859.3409","0.194",1],["3859","5",1],["3858.827","0.0521",1],["3858.8208","0.001",1],["3858.679","0.26",1],["3858.4814","0.07477305",1],["3858.1669","1.03503422",1],["3857.6005","0.006",1],["3857.4005","0.004",1],["3857.2005","0.004",1],["3857.1871","1.218",1],["3857.0005","0.004",1],["3856.8135","0.0646",1],["3856.8005","0.004",1],["3856.2412","0.001",1],["3856.2349","1.03503422",1],["3856.0197","0.01037339",1],["3855.8781","0.23178117",1],["3855.8005","0.004",1],["3855.7165","0.00259355",1],["3855.4858","0.25875855",1],["3854.4584","0.01",1],["3853.6616","0.001",1],["3853.1373","0.92",1],["3852.5072","0.48599702",1],["3851.3926","0.13008333",1],["3851.082","0.001",1],["3850.9317","2",1],["3850.6359","0.34770165",1],["3850.2058","0.51751624",1],["3850.0823","0.15",1],["3850.0042","0.5175171",1],["3850","0.001",1],["3849.6325","1.8",1],["3849.41","0.3",1],["3848.9686","1.85",1],["3848.7426","0.18511466",1],["3848.52","0.3",1],["3848.5024","0.001",1],["3848.42","0.3",1],["3848.1618","2.204",1],["3847.77","0.3",1],["3847.48","0.3",1],["3847.3581","2.05",1],["3846.8259","0.0646",1],["3846.59","0.3",1],["3846.49","0.3",1],["3845.9228","0.001",1],["3844.184","0.00260133",1],["3844.0092","6.3",1],["3843.3432","0.001",1],["3841","0.06300963",1],["3840.7636","0.001",1],["3840","0.201",3],["3839.7681","18",1],["3839.5328","0.05214011",1],["3838.184","0.001",1],["3837.2344","0.27589557",1],["3836.6479","5.2",1],["3836","2.37196773",3],["3835.6044","0.001",1],["3833.6053","0.25873556",1],["3833.0248","0.001",1],["3833","0.8726502",1],["3832.6859","0.00260913",1],["3832","0.007",1],["3831.637","6",1],["3831.0602","0.001",1],["3830.4452","0.001",1],["3830","0.20375718",4],["3829.7125","0.07833486",1],["3829.6283","0.3519681",1],["3829","0.0039261",1],["3827.8656","0.001",1],["3826.0001","0.53251232",1],["3826","0.0509",1],["3825.7834","0.00698562",1],["3825.286","0.001",1],["3823.0001","0.03010127",1],["3822.8014","0.00261588",1],["3822.7064","0.001",1],["3822.2","1",1],["3822.1121","0.35994101",1],["3821.2222","0.00261696",1],["3821","0.001",1],["3820.1268","0.001",1],["3820","1.12992803",4],["3819","0.01331195",2],["3817.5472","0.001",1],["3816","1.13807184",2],["3815.8343","0.32463428",1],["3815.7834","0.00525295",1],["3815","28.99386799",4],["3814.9676","0.001",1],["3813","0.91303023",4],["3812.388","0.002",2],["3811.2257","0.07",1],["3810","0.32573997",2],["3809.8084","0.001",1],["3809.7928","0.00262481",1],["3807.2288","0.001",1],["3806.8421","0.07003461",1],["3806","0.19",1],["3805.8041","0.05678805",1],["3805","1.01",2],["3804.6492","0.001",1],["3804.3551","0.1",1],["3803","0.005",1],["3802.22","2.05042631",1],["3802.0696","0.001",1],["3802","1.63290092",1],["3801.2257","0.07",1],["3801","57.4",3],["3800.9853","0.02492278",1],["3800.8421","0.06503533",1],["3800.7844","0.02812628",1],["3800.0001","0.00409473",1],["3800","17.91401074",15],["3799.49","0.001",1],["3799","0.1",1],["3796.9104","0.001",1],["3796","9.00128053",2],["3795.5441","0.0028",1],["3794.3308","0.001",1],["3791","55",1],["3790.7777","0.07",1],["3790","12.03238184",7],["3789","1",1],["3788","0.21110454",2],["3787.2959","9",1],["3786.592","0.001",1],["3786","9.01916822",2],["3785","12.87914268",5],["3784.0124","0.001",1],["3781.4328","0.002",2],["3781","56.3",2],["3780.7777","0.07",1],["3780","23.41537654",10],["3778.8532","0.002",2],["3776","9",1],["3774","0.003",1],["3772.2481","0.06901672",1],["3771","55.1",2],["3770.7777","0.07",1],["3770","7.30268416",5],["3769","0.25",1],["3768","1.3725",3],["3766.66","0.02",1],["3766","7.64837924",2],["3765.58","1.22775492",1],["3762.58","1.22873383",1],["3761","51.68262164",1],["3760.8031","0.0399",1],["3760.7777","0.07",1]],"timestamp":"2019-03-06T23:19:17.705Z","checksum":-1785549915}]}` update := `{"table":"spot/depth","action":"update","data":[{"instrument_id":"BTC-USDT","asks":[["3864.6786","0",0],["3864.9852","0",0],["3865.9994","0.48402971",1],["3866.4004","0.001",1],["3866.7995","0.3273",2],["3867.4566","0",0],["3867.7031","0.025",1],["3868.0436","0",0],["3868.346","0",0],["3868.3695","0.051",1],["3870.9243","0.642",1],["3874.9942","0.51751796",1],["3875.7057","0",0],["3939","0.001",1]],"bids":[["3864.55","0.0565449",1],["3863.8282","0",0],["3863.8153","0",0],["3863.7898","0.01320077",1],["3863.4807","0.02112123",1],["3863.3002","0.04233533",1],["3863.1717","0.03379397",1],["3863.0685","0.04438179",1],["3863.0286","0.7362564",1],["3862.9912","0.06773651",1],["3862.8626","0.05407035",1],["3862.7595","0.07101087",1],["3862.313","0.3756",2],["3862.1848","0.012",1],["3862.0734","0",0],["3861.8391","0.025",1],["3861.7888","0",0],["3856.6716","0.38893641",1],["3768","0",0],["3766.66","0",0],["3766","0",0],["3765.58","0",0],["3762.58","0",0],["3761","0",0],["3760.8031","0",0],["3760.7777","0",0]],"timestamp":"2019-03-06T23:19:18.239Z","checksum":-1587788848}]}` var dataResponse okgroup.WebsocketDataResponse @@ -1675,9 +1673,6 @@ func TestOrderBookUpdateChecksumCalculator(t *testing.T) { // TestOrderBookUpdateChecksumCalculatorWithDash logic test func TestOrderBookUpdateChecksumCalculatorWith8DecimalPlaces(t *testing.T) { TestSetDefaults(t) - if !websocketEnabled { - t.Skip("Websocket not enabled, skipping") - } original := `{"table":"spot/depth","action":"partial","data":[{"instrument_id":"WAVES-BTC","asks":[["0.000714","1.15414979",1],["0.000715","3.3",2],["0.000717","426.71348",2],["0.000719","140.84507042",1],["0.00072","590.77",1],["0.000721","991.77",1],["0.000724","0.3532032",1],["0.000725","58.82698567",1],["0.000726","1033.15469748",2],["0.000729","0.35320321",1],["0.00073","352.77",1],["0.000735","0.38469748",1],["0.000736","625.77",1],["0.00075191","152.44796961",1],["0.00075192","114.3359772",1],["0.00075193","85.7519829",1],["0.00075194","64.31398718",1],["0.00075195","48.23549038",1],["0.00075196","36.17661779",1],["0.00075199","61.04804253",1],["0.0007591","70.71318474",1],["0.0007621","53.03488855",1],["0.00076211","39.77616642",1],["0.00076212","29.83212481",1],["0.0007635","22.37409361",1],["0.00076351","29.36599786",2],["0.00076352","9.43907074",1],["0.00076353","7.07930306",1],["0.00076354","14.15860612",1],["0.00076355","3.53965153",1],["0.00076369","3.53965153",1],["0.0008","34.36841101",1],["0.00082858","1.69936503",1],["0.00083232","2.8",1],["0.00084","15.69220129",1],["0.00085","4.42785042",1],["0.00088","0.1",1],["0.000891","0.1",1],["0.0009","12.41486491",2],["0.00093","5",1],["0.0012","12.31486492",1],["0.00531314","6.91803114",1],["0.00799999","0.02",1],["0.0084","0.05989",1],["0.00931314","5.18852336",1],["0.0799999","0.02",1],["0.499","6.00423396",1],["0.5","0.4995",1],["0.799999","0.02",1],["4.99","2",1],["5","3.98583144",1],["7.99999999","0.02",1],["79.99999999","0.02",1],["799.99999999","0.02986704",1]],"bids":[["0.000709","222.91679881",3],["0.000703","0.47161952",1],["0.000701","140.73015789",2],["0.0007","0.3",1],["0.000699","401",1],["0.000698","232.61801667",2],["0.000689","0.71396896",1],["0.000688","0.69910125",1],["0.000613","227.54771052",1],["0.0005","0.01",1],["0.00026789","3.69905341",1],["0.000238","2.4",1],["0.00022","0.53",1],["0.0000055","374.09871696",1],["0.00000056","222",1],["0.00000055","736.84761363",1],["0.0000002","999",1],["0.00000009","1222.22222417",1],["0.00000008","20868.64520447",1],["0.00000002","110000",1],["0.00000001","10000",1]],"timestamp":"2019-03-12T22:22:42.274Z","checksum":1319037905}]}` update := `{"table":"spot/depth","action":"update","data":[{"instrument_id":"WAVES-BTC","asks":[["0.000715","100.48199596",3],["0.000716","62.21679881",1]],"bids":[["0.000713","38.95772168",1]],"timestamp":"2019-03-12T22:22:42.938Z","checksum":-131160897}]}` var dataResponse okgroup.WebsocketDataResponse diff --git a/exchanges/okgroup/okgroup.go b/exchanges/okgroup/okgroup.go index 2c47f805..e931c06e 100644 --- a/exchanges/okgroup/okgroup.go +++ b/exchanges/okgroup/okgroup.go @@ -16,7 +16,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -160,6 +160,13 @@ func (o *OKGroup) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + o.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + false, + false, + false, + false, + exch.Name) } } diff --git a/exchanges/okgroup/okgroup_websocket.go b/exchanges/okgroup/okgroup_websocket.go index 98cb09d5..bbabfc47 100644 --- a/exchanges/okgroup/okgroup_websocket.go +++ b/exchanges/okgroup/okgroup_websocket.go @@ -5,7 +5,6 @@ import ( "fmt" "hash/crc32" "net/http" - "sort" "strconv" "strings" "sync" @@ -16,7 +15,8 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -465,7 +465,7 @@ func (o *OKGroup) WsProcessPartialOrderBook(wsEventData *WebsocketDataWrapper, i ExchangeName: o.GetName(), } - err := o.Websocket.Orderbook.LoadSnapshot(&newOrderBook, o.GetName(), true) + err := o.Websocket.Orderbook.LoadSnapshot(&newOrderBook, true) if err != nil { return err } @@ -480,43 +480,29 @@ func (o *OKGroup) WsProcessPartialOrderBook(wsEventData *WebsocketDataWrapper, i // WsProcessUpdateOrderbook updates an existing orderbook using websocket data // After merging WS data, it will sort, validate and finally update the existing orderbook func (o *OKGroup) WsProcessUpdateOrderbook(wsEventData *WebsocketDataWrapper, instrument currency.Pair, tableName string) error { - internalOrderbook, err := o.GetOrderbookEx(instrument, o.GetAssetTypeFromTableName(tableName)) + update := wsorderbook.WebsocketOrderbookUpdate{ + AssetType: orderbook.Spot, + CurrencyPair: instrument, + UpdateTime: wsEventData.Timestamp, + } + update.Asks = o.AppendWsOrderbookItems(wsEventData.Asks) + update.Bids = o.AppendWsOrderbookItems(wsEventData.Bids) + err := o.Websocket.Orderbook.Update(&update) if err != nil { - return errors.New("orderbook nil, could not load existing orderbook") + log.Error(err) } - if internalOrderbook.LastUpdated.After(wsEventData.Timestamp) { - if o.Verbose { - log.Errorf("Orderbook update out of order. Existing: %v, Attempted: %v", internalOrderbook.LastUpdated.Unix(), wsEventData.Timestamp.Unix()) - } - return errors.New("updated orderbook is older than existing") - } - internalOrderbook.Asks = o.WsUpdateOrderbookEntry(wsEventData.Asks, internalOrderbook.Asks) - internalOrderbook.Bids = o.WsUpdateOrderbookEntry(wsEventData.Bids, internalOrderbook.Bids) - sort.Slice(internalOrderbook.Asks, func(i, j int) bool { - return internalOrderbook.Asks[i].Price < internalOrderbook.Asks[j].Price - }) - sort.Slice(internalOrderbook.Bids, func(i, j int) bool { - return internalOrderbook.Bids[i].Price > internalOrderbook.Bids[j].Price - }) - checksum := o.CalculateUpdateOrderbookChecksum(&internalOrderbook) + updatedOb := o.Websocket.Orderbook.GetOrderbook(instrument, orderbook.Spot) + checksum := o.CalculateUpdateOrderbookChecksum(updatedOb) if checksum == wsEventData.Checksum { if o.Verbose { log.Debug("Orderbook valid") } - internalOrderbook.LastUpdated = wsEventData.Timestamp - if o.Verbose { - log.Debug("Internalising orderbook") - } - - err := o.Websocket.Orderbook.LoadSnapshot(&internalOrderbook, o.GetName(), true) - if err != nil { - log.Error(err) - } o.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: o.GetName(), Asset: o.GetAssetTypeFromTableName(tableName), Pair: instrument, } + } else { if o.Verbose { log.Debug("Orderbook invalid") @@ -526,35 +512,6 @@ func (o *OKGroup) WsProcessUpdateOrderbook(wsEventData *WebsocketDataWrapper, in return nil } -// WsUpdateOrderbookEntry takes WS bid or ask data and merges it with existing orderbook bid or ask data -func (o *OKGroup) WsUpdateOrderbookEntry(wsEntries [][]interface{}, existingOrderbookEntries []orderbook.Item) []orderbook.Item { - for j := range wsEntries { - wsEntryPrice, _ := strconv.ParseFloat(wsEntries[j][0].(string), 64) - wsEntryAmount, _ := strconv.ParseFloat(wsEntries[j][1].(string), 64) - matchFound := false - for k := 0; k < len(existingOrderbookEntries); k++ { - if existingOrderbookEntries[k].Price != wsEntryPrice { - continue - } - matchFound = true - if wsEntryAmount == 0 { - existingOrderbookEntries = append(existingOrderbookEntries[:k], existingOrderbookEntries[k+1:]...) - k-- - continue - } - existingOrderbookEntries[k].Amount = wsEntryAmount - continue - } - if !matchFound { - existingOrderbookEntries = append(existingOrderbookEntries, orderbook.Item{ - Amount: wsEntryAmount, - Price: wsEntryPrice, - }) - } - } - return existingOrderbookEntries -} - // CalculatePartialOrderbookChecksum alternates over the first 25 bid and ask entries from websocket data // The checksum is made up of the price and the quantity with a semicolon (:) deliminating them // This will also work when there are less than 25 entries (for whatever reason) diff --git a/exchanges/okgroup/okgroup_wrapper.go b/exchanges/okgroup/okgroup_wrapper.go index c7edd22d..6f95e380 100644 --- a/exchanges/okgroup/okgroup_wrapper.go +++ b/exchanges/okgroup/okgroup_wrapper.go @@ -11,7 +11,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/poloniex/poloniex.go b/exchanges/poloniex/poloniex.go index 62b02a59..18f25fee 100644 --- a/exchanges/poloniex/poloniex.go +++ b/exchanges/poloniex/poloniex.go @@ -16,7 +16,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -95,6 +95,7 @@ func (p *Poloniex) SetDefaults() { wshandler.WebsocketAuthenticatedEndpointsSupported p.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit p.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + p.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets user exchange configuration settings @@ -155,6 +156,13 @@ func (p *Poloniex) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + p.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + true, + true, + true, + false, + exch.Name) } } diff --git a/exchanges/poloniex/poloniex_test.go b/exchanges/poloniex/poloniex_test.go index 4c58deee..63669c51 100644 --- a/exchanges/poloniex/poloniex_test.go +++ b/exchanges/poloniex/poloniex_test.go @@ -11,7 +11,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/sharedtestvalues" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) var p Poloniex diff --git a/exchanges/poloniex/poloniex_websocket.go b/exchanges/poloniex/poloniex_websocket.go index dbc09622..04739489 100644 --- a/exchanges/poloniex/poloniex_websocket.go +++ b/exchanges/poloniex/poloniex_websocket.go @@ -13,7 +13,8 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -150,12 +151,12 @@ func (p *Poloniex) WsHandleData() { p.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: p.GetName(), - Asset: "SPOT", + Asset: orderbook.Spot, Pair: currency.NewPairFromString(currencyPair), } case "o": currencyPair := CurrencyPairID[chanID] - err := p.WsProcessOrderbookUpdate(dataL3, currencyPair) + err := p.WsProcessOrderbookUpdate(int64(data[1].(float64)), dataL3, currencyPair) if err != nil { p.Websocket.DataHandler <- err continue @@ -163,7 +164,7 @@ func (p *Poloniex) WsHandleData() { p.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Exchange: p.GetName(), - Asset: "SPOT", + Asset: orderbook.Spot, Pair: currency.NewPairFromString(currencyPair), } case "t": @@ -217,7 +218,7 @@ func (p *Poloniex) wsHandleTickerData(data []interface{}) { p.Websocket.DataHandler <- wshandler.TickerData{ Timestamp: time.Now(), Exchange: p.GetName(), - AssetType: "SPOT", + AssetType: orderbook.Spot, LowPrice: t.LowestAsk, HighPrice: t.HighestBid, } @@ -321,43 +322,35 @@ func (p *Poloniex) WsProcessOrderbookSnapshot(ob []interface{}, symbol string) e var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = "SPOT" + newOrderBook.AssetType = orderbook.Spot newOrderBook.Pair = currency.NewPairFromString(symbol) - return p.Websocket.Orderbook.LoadSnapshot(&newOrderBook, p.GetName(), false) + return p.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false) } -// WsProcessOrderbookUpdate processses new orderbook updates -func (p *Poloniex) WsProcessOrderbookUpdate(target []interface{}, symbol string) error { +// WsProcessOrderbookUpdate processes new orderbook updates +func (p *Poloniex) WsProcessOrderbookUpdate(sequenceNumber int64, target []interface{}, symbol string) error { sideCheck := target[1].(float64) - cP := currency.NewPairFromString(symbol) - price, err := strconv.ParseFloat(target[2].(string), 64) if err != nil { return err } - volume, err := strconv.ParseFloat(target[3].(string), 64) if err != nil { return err } - - if sideCheck == 0 { - return p.Websocket.Orderbook.Update(nil, - []orderbook.Item{{Price: price, Amount: volume}}, - cP, - time.Now(), - p.GetName(), - "SPOT") + update := &wsorderbook.WebsocketOrderbookUpdate{ + CurrencyPair: cP, + AssetType: orderbook.Spot, + UpdateID: sequenceNumber, } - - return p.Websocket.Orderbook.Update([]orderbook.Item{{Price: price, Amount: volume}}, - nil, - cP, - time.Now(), - p.GetName(), - "SPOT") + if sideCheck == 0 { + update.Bids = []orderbook.Item{{Price: price, Amount: volume}} + } else { + update.Asks = []orderbook.Item{{Price: price, Amount: volume}} + } + return p.Websocket.Orderbook.Update(update) } // CurrencyPairID contains a list of IDS for currency pairs. diff --git a/exchanges/poloniex/poloniex_wrapper.go b/exchanges/poloniex/poloniex_wrapper.go index 892f580b..f88061f2 100644 --- a/exchanges/poloniex/poloniex_wrapper.go +++ b/exchanges/poloniex/poloniex_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/sharedtestvalues/sharedtestvalues.go b/exchanges/sharedtestvalues/sharedtestvalues.go index a268282e..3e9c92c8 100644 --- a/exchanges/sharedtestvalues/sharedtestvalues.go +++ b/exchanges/sharedtestvalues/sharedtestvalues.go @@ -12,7 +12,7 @@ const ( WebsocketResponseExtendedTimeout = (15 * time.Second) // WebsocketChannelOverrideCapacity used in websocket testing // Defines channel capacity as defaults size can block tests - WebsocketChannelOverrideCapacity = 10 + WebsocketChannelOverrideCapacity = 20 ) // GetWebsocketInterfaceChannelOverride returns a new interface based channel diff --git a/exchanges/wshandler/websocket.go b/exchanges/websocket/wshandler/wshandler.go similarity index 74% rename from exchanges/wshandler/websocket.go rename to exchanges/websocket/wshandler/wshandler.go index 3a3c8b65..debfd55d 100644 --- a/exchanges/wshandler/websocket.go +++ b/exchanges/websocket/wshandler/wshandler.go @@ -1,15 +1,21 @@ package wshandler import ( + "bytes" + "compress/flate" + "compress/gzip" "errors" "fmt" + "io/ioutil" + "net/http" + "net/url" "strings" "sync" "time" + "github.com/gorilla/websocket" + "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" - "github.com/thrasher-corp/gocryptotrader/currency" - "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -95,15 +101,15 @@ func (w *Websocket) Connect() error { go w.trafficMonitor(&anotherWG) anotherWG.Wait() if !w.connectionMonitorRunning { - go w.wsConnectionMonitor() + go w.connectionMonitor() } go w.manageSubscriptions() return nil } -// WsConnectionMonitor ensures that the WS keeps connecting -func (w *Websocket) wsConnectionMonitor() { +// connectionMonitor ensures that the WS keeps connecting +func (w *Websocket) connectionMonitor() { w.m.Lock() w.connectionMonitorRunning = true w.m.Unlock() @@ -116,13 +122,13 @@ func (w *Websocket) wsConnectionMonitor() { w.m.Lock() if !w.enabled { w.m.Unlock() - w.DataHandler <- fmt.Errorf("%v WsConnectionMonitor: websocket disabled, shutting down", w.exchangeName) + w.DataHandler <- fmt.Errorf("%v connectionMonitor: websocket disabled, shutting down", w.exchangeName) err := w.Shutdown() if err != nil { log.Error(err) } if w.verbose { - log.Debugf("%v WsConnectionMonitor exiting", w.exchangeName) + log.Debugf("%v connectionMonitor exiting", w.exchangeName) } return } @@ -198,7 +204,6 @@ func (w *Websocket) Shutdown() error { if !w.connected && w.ShutdownC == nil { return fmt.Errorf("%v cannot shutdown a disconnected websocket", w.exchangeName) } - if w.verbose { log.Debugf("%v shutting down websocket channels", w.exchangeName) } @@ -225,11 +230,11 @@ func (w *Websocket) Shutdown() error { } // WebsocketReset sends the shutdown command, waits for channel/func closure and then reconnects -func (w *Websocket) WebsocketReset() error { +func (w *Websocket) WebsocketReset() { err := w.Shutdown() if err != nil { // does not return here to allow connection to be made if already shut down - log.Errorf("%v shutdown error: %v", w.exchangeName, err) + w.DataHandler <- fmt.Errorf("%v shutdown error: %v", w.exchangeName, err) } log.Infof("%v reconnecting to websocket", w.exchangeName) w.m.Lock() @@ -237,9 +242,8 @@ func (w *Websocket) WebsocketReset() error { w.m.Unlock() err = w.Connect() if err != nil { - log.Errorf("%v connection error: %v", w.exchangeName, err) + w.DataHandler <- fmt.Errorf("%v connection error: %v", w.exchangeName, err) } - return err } // trafficMonitor monitors traffic and switches connection modes for websocket @@ -343,7 +347,6 @@ func (w *Websocket) SetWsStatusAndConnection(enabled bool) error { enabled) } w.enabled = enabled - if !w.init { if enabled { if w.connected { @@ -419,216 +422,6 @@ func (w *Websocket) GetName() string { return w.exchangeName } -// Update updates a local cache using bid targets and ask targets then updates -// main cache in orderbook.go -// Volume == 0; deletion at price target -// Price target not found; append of price target -// Price target found; amend volume of price target -func (w *WebsocketOrderbookLocal) Update(bidTargets, askTargets []orderbook.Item, - p currency.Pair, - updated time.Time, - exchName, assetType string) error { - if bidTargets == nil && askTargets == nil { - return errors.New("exchange.go websocket orderbook cache Update() error - cannot have bids and ask targets both nil") - } - - if w.lastUpdated.After(updated) { - return errors.New("exchange.go WebsocketOrderbookLocal Update() - update is before last update time") - } - - w.m.Lock() - defer w.m.Unlock() - - var orderbookAddress *orderbook.Base - for i := range w.ob { - if w.ob[i].Pair == p && w.ob[i].AssetType == assetType { - orderbookAddress = w.ob[i] - } - } - - if orderbookAddress == nil { - return fmt.Errorf("exchange.go WebsocketOrderbookLocal Update() - orderbook.Base could not be found for Exchange %s CurrencyPair: %s AssetType: %s", - exchName, - p.String(), - assetType) - } - - if len(orderbookAddress.Asks) == 0 || len(orderbookAddress.Bids) == 0 { - return errors.New("exchange.go websocket orderbook cache Update() error - snapshot incorrectly loaded") - } - - if orderbookAddress.Pair == (currency.Pair{}) { - return fmt.Errorf("exchange.go websocket orderbook cache Update() error - snapshot not found %v", - p) - } - - for x := range bidTargets { - // bid targets - func() { - for y := range orderbookAddress.Bids { - if orderbookAddress.Bids[y].Price == bidTargets[x].Price { - if bidTargets[x].Amount == 0 { - // Delete - orderbookAddress.Bids = append(orderbookAddress.Bids[:y], - orderbookAddress.Bids[y+1:]...) - return - } - // Amend - orderbookAddress.Bids[y].Amount = bidTargets[x].Amount - return - } - } - - if bidTargets[x].Amount == 0 { - // Makes sure we dont append things we missed - return - } - - // Append - orderbookAddress.Bids = append(orderbookAddress.Bids, orderbook.Item{ - Price: bidTargets[x].Price, - Amount: bidTargets[x].Amount, - }) - }() - // bid targets - } - - for x := range askTargets { - func() { - for y := range orderbookAddress.Asks { - if orderbookAddress.Asks[y].Price == askTargets[x].Price { - if askTargets[x].Amount == 0 { - // Delete - orderbookAddress.Asks = append(orderbookAddress.Asks[:y], - orderbookAddress.Asks[y+1:]...) - return - } - // Amend - orderbookAddress.Asks[y].Amount = askTargets[x].Amount - return - } - } - - if askTargets[x].Amount == 0 { - // Makes sure we dont append things we missed - return - } - - // Append - orderbookAddress.Asks = append(orderbookAddress.Asks, orderbook.Item{ - Price: askTargets[x].Price, - Amount: askTargets[x].Amount, - }) - }() - } - - return orderbookAddress.Process() - -} - -// LoadSnapshot loads initial snapshot of orderbook data, overite allows full -// orderbook to be completely rewritten because the exchange is a doing a full -// update not an incremental one -func (w *WebsocketOrderbookLocal) LoadSnapshot(newOrderbook *orderbook.Base, exchName string, overwrite bool) error { - if len(newOrderbook.Asks) == 0 || len(newOrderbook.Bids) == 0 { - return errors.New("exchange.go websocket orderbook cache LoadSnapshot() error - snapshot ask and bids are nil") - } - - w.m.Lock() - defer w.m.Unlock() - - for i := range w.ob { - if w.ob[i].Pair.Equal(newOrderbook.Pair) && w.ob[i].AssetType == newOrderbook.AssetType { - if overwrite { - w.ob[i] = newOrderbook - return newOrderbook.Process() - } - return errors.New("exchange.go websocket orderbook cache LoadSnapshot() error - Snapshot instance already found") - } - } - - w.ob = append(w.ob, newOrderbook) - return newOrderbook.Process() -} - -// UpdateUsingID updates orderbooks using specified ID -func (w *WebsocketOrderbookLocal) UpdateUsingID(bidTargets, askTargets []orderbook.Item, - p currency.Pair, - exchName, assetType, action string) error { - w.m.Lock() - defer w.m.Unlock() - - var orderbookAddress *orderbook.Base - for i := range w.ob { - if w.ob[i].Pair == p && w.ob[i].AssetType == assetType { - orderbookAddress = w.ob[i] - } - } - - if orderbookAddress == nil { - return fmt.Errorf("exchange.go WebsocketOrderbookLocal Update() - orderbook.Base could not be found for Exchange %s CurrencyPair: %s AssetType: %s", - exchName, - assetType, - p.String()) - } - - switch action { - case "update": - for _, target := range bidTargets { - for i := range orderbookAddress.Bids { - if orderbookAddress.Bids[i].ID == target.ID { - orderbookAddress.Bids[i].Amount = target.Amount - break - } - } - } - - for _, target := range askTargets { - for i := range orderbookAddress.Asks { - if orderbookAddress.Asks[i].ID == target.ID { - orderbookAddress.Asks[i].Amount = target.Amount - break - } - } - } - - case "delete": - for _, target := range bidTargets { - for i := range orderbookAddress.Bids { - if orderbookAddress.Bids[i].ID == target.ID { - orderbookAddress.Bids = append(orderbookAddress.Bids[:i], - orderbookAddress.Bids[i+1:]...) - break - } - } - } - - for _, target := range askTargets { - for i := range orderbookAddress.Asks { - if orderbookAddress.Asks[i].ID == target.ID { - orderbookAddress.Asks = append(orderbookAddress.Asks[:i], - orderbookAddress.Asks[i+1:]...) - break - } - } - } - - case "insert": - orderbookAddress.Bids = append(orderbookAddress.Bids, bidTargets...) - orderbookAddress.Asks = append(orderbookAddress.Asks, askTargets...) - } - - return orderbookAddress.Process() -} - -// FlushCache flushes w.ob data to be garbage collected and refreshed when a -// connection is lost and reconnected -func (w *WebsocketOrderbookLocal) FlushCache() { - w.m.Lock() - w.ob = nil - w.m.Unlock() -} - // GetFunctionality returns a functionality bitmask for the websocket // connection func (w *Websocket) GetFunctionality() uint32 { @@ -721,9 +514,10 @@ func (w *Websocket) SetChannelUnsubscriber(unsubscriber func(channelToUnsubscrib } // ManageSubscriptions ensures the subscriptions specified continue to be subscribed to -func (w *Websocket) manageSubscriptions() error { +func (w *Websocket) manageSubscriptions() { if !w.SupportsFunctionality(WebsocketSubscribeSupported) && !w.SupportsFunctionality(WebsocketUnsubscribeSupported) { - return fmt.Errorf("%v does not support channel subscriptions, exiting ManageSubscriptions()", w.exchangeName) + w.DataHandler <- fmt.Errorf("%v does not support channel subscriptions, exiting ManageSubscriptions()", w.exchangeName) + return } w.Wg.Add(1) defer func() { @@ -739,7 +533,7 @@ func (w *Websocket) manageSubscriptions() error { if w.verbose { log.Debugf("%v shutdown manageSubscriptions", w.exchangeName) } - return nil + return default: time.Sleep(manageSubscriptionsDelay) if w.verbose { @@ -908,3 +702,163 @@ func (w *Websocket) CanUseAuthenticatedEndpoints() bool { defer w.subscriptionLock.Unlock() return w.canUseAuthenticatedEndpoints } + +// AddResponseWithID adds data to IDResponses with locks and a nil check +func (w *WebsocketConnection) AddResponseWithID(id int64, data []byte) { + w.Lock() + defer w.Unlock() + if w.IDResponses == nil { + w.IDResponses = make(map[int64][]byte) + } + w.IDResponses[id] = data +} + +// Dial sets proxy urls and then connects to the websocket +func (w *WebsocketConnection) Dial(dialer *websocket.Dialer, headers http.Header) error { + if w.ProxyURL != "" { + proxy, err := url.Parse(w.ProxyURL) + if err != nil { + return err + } + dialer.Proxy = http.ProxyURL(proxy) + } + + var err error + var conStatus *http.Response + w.Connection, conStatus, err = dialer.Dial(w.URL, headers) + if err != nil { + if conStatus != nil { + return fmt.Errorf("%v %v %v Error: %v", w.URL, conStatus, conStatus.StatusCode, err) + } + return fmt.Errorf("%v Error: %v", w.URL, err) + } + return nil +} + +// SendMessage the one true message request. Sends message to WS +func (w *WebsocketConnection) SendMessage(data interface{}) error { + w.Lock() + defer w.Unlock() + json, err := common.JSONEncode(data) + if err != nil { + return err + } + if w.Verbose { + log.Debugf("%v sending message to websocket %v", w.ExchangeName, string(json)) + } + if w.RateLimit > 0 { + time.Sleep(time.Duration(w.RateLimit) * time.Millisecond) + } + return w.Connection.WriteMessage(websocket.TextMessage, json) +} + +// SendMessageReturnResponse will send a WS message to the connection +// It will then run a goroutine to await a JSON response +// If there is no response it will return an error +func (w *WebsocketConnection) SendMessageReturnResponse(id int64, request interface{}) ([]byte, error) { + err := w.SendMessage(request) + if err != nil { + return nil, err + } + var wg sync.WaitGroup + wg.Add(1) + go w.WaitForResult(id, &wg) + defer func() { + delete(w.IDResponses, id) + }() + wg.Wait() + if _, ok := w.IDResponses[id]; !ok { + return nil, fmt.Errorf("timeout waiting for response with ID %v", id) + } + + return w.IDResponses[id], nil +} + +// WaitForResult will keep checking w.IDResponses for a response ID +// If the timer expires, it will return without +func (w *WebsocketConnection) WaitForResult(id int64, wg *sync.WaitGroup) { + defer wg.Done() + timer := time.NewTimer(w.ResponseMaxLimit) + for { + select { + case <-timer.C: + return + default: + w.Lock() + for k := range w.IDResponses { + if k == id { + w.Unlock() + return + } + } + w.Unlock() + time.Sleep(w.ResponseCheckTimeout) + } + } +} + +// ReadMessage reads messages, can handle text, gzip and binary +func (w *WebsocketConnection) ReadMessage() (WebsocketResponse, error) { + mType, resp, err := w.Connection.ReadMessage() + if err != nil { + return WebsocketResponse{}, err + } + var standardMessage []byte + switch mType { + case websocket.TextMessage: + standardMessage = resp + case websocket.BinaryMessage: + standardMessage, err = w.parseBinaryResponse(resp) + if err != nil { + return WebsocketResponse{}, err + } + } + if w.Verbose { + log.Debugf("%v Websocket message received: %v", + w.ExchangeName, + string(standardMessage)) + } + return WebsocketResponse{Raw: standardMessage, Type: mType}, nil +} + +// parseBinaryResponse parses a websocket binary response into a usable byte array +func (w *WebsocketConnection) parseBinaryResponse(resp []byte) ([]byte, error) { + var standardMessage []byte + var err error + // Detect GZIP + if resp[0] == 31 && resp[1] == 139 { + b := bytes.NewReader(resp) + var gReader *gzip.Reader + gReader, err = gzip.NewReader(b) + if err != nil { + return standardMessage, err + } + standardMessage, err = ioutil.ReadAll(gReader) + if err != nil { + return standardMessage, err + } + err = gReader.Close() + if err != nil { + return standardMessage, err + } + } else { + reader := flate.NewReader(bytes.NewReader(resp)) + standardMessage, err = ioutil.ReadAll(reader) + if err != nil { + return standardMessage, err + } + err = reader.Close() + if err != nil { + return standardMessage, err + } + } + return standardMessage, nil +} + +// GenerateMessageID Creates a messageID to checkout +func (w *WebsocketConnection) GenerateMessageID(useNano bool) int64 { + if useNano { + return time.Now().UnixNano() + } + return time.Now().Unix() +} diff --git a/exchanges/wshandler/websocket_test.go b/exchanges/websocket/wshandler/wshandler_test.go similarity index 62% rename from exchanges/wshandler/websocket_test.go rename to exchanges/websocket/wshandler/wshandler_test.go index 5cf62409..74b21ce6 100644 --- a/exchanges/wshandler/websocket_test.go +++ b/exchanges/websocket/wshandler/wshandler_test.go @@ -5,9 +5,6 @@ import ( "strings" "testing" "time" - - "github.com/thrasher-corp/gocryptotrader/currency" - "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" ) var ws *Websocket @@ -124,185 +121,6 @@ func TestWebsocket(t *testing.T) { } } -func TestInsertingSnapShots(t *testing.T) { - var snapShot1 orderbook.Base - asks := []orderbook.Item{ - {Price: 6000, Amount: 1, ID: 1}, - {Price: 6001, Amount: 0.5, ID: 2}, - {Price: 6002, Amount: 2, ID: 3}, - {Price: 6003, Amount: 3, ID: 4}, - {Price: 6004, Amount: 5, ID: 5}, - {Price: 6005, Amount: 2, ID: 6}, - {Price: 6006, Amount: 1.5, ID: 7}, - {Price: 6007, Amount: 0.5, ID: 8}, - {Price: 6008, Amount: 23, ID: 9}, - {Price: 6009, Amount: 9, ID: 10}, - {Price: 6010, Amount: 7, ID: 11}, - } - - bids := []orderbook.Item{ - {Price: 5999, Amount: 1, ID: 12}, - {Price: 5998, Amount: 0.5, ID: 13}, - {Price: 5997, Amount: 2, ID: 14}, - {Price: 5996, Amount: 3, ID: 15}, - {Price: 5995, Amount: 5, ID: 16}, - {Price: 5994, Amount: 2, ID: 17}, - {Price: 5993, Amount: 1.5, ID: 18}, - {Price: 5992, Amount: 0.5, ID: 19}, - {Price: 5991, Amount: 23, ID: 20}, - {Price: 5990, Amount: 9, ID: 21}, - {Price: 5989, Amount: 7, ID: 22}, - } - - snapShot1.Asks = asks - snapShot1.Bids = bids - snapShot1.AssetType = "SPOT" - snapShot1.Pair = currency.NewPairFromString("BTCUSD") - - ws.Orderbook.LoadSnapshot(&snapShot1, "ExchangeTest", false) - - var snapShot2 orderbook.Base - asks = []orderbook.Item{ - {Price: 51, Amount: 1, ID: 1}, - {Price: 52, Amount: 0.5, ID: 2}, - {Price: 53, Amount: 2, ID: 3}, - {Price: 54, Amount: 3, ID: 4}, - {Price: 55, Amount: 5, ID: 5}, - {Price: 56, Amount: 2, ID: 6}, - {Price: 57, Amount: 1.5, ID: 7}, - {Price: 58, Amount: 0.5, ID: 8}, - {Price: 59, Amount: 23, ID: 9}, - {Price: 50, Amount: 9, ID: 10}, - {Price: 60, Amount: 7, ID: 11}, - } - - bids = []orderbook.Item{ - {Price: 49, Amount: 1, ID: 12}, - {Price: 48, Amount: 0.5, ID: 13}, - {Price: 47, Amount: 2, ID: 14}, - {Price: 46, Amount: 3, ID: 15}, - {Price: 45, Amount: 5, ID: 16}, - {Price: 44, Amount: 2, ID: 17}, - {Price: 43, Amount: 1.5, ID: 18}, - {Price: 42, Amount: 0.5, ID: 19}, - {Price: 41, Amount: 23, ID: 20}, - {Price: 40, Amount: 9, ID: 21}, - {Price: 39, Amount: 7, ID: 22}, - } - - snapShot2.Asks = asks - snapShot2.Bids = bids - snapShot2.AssetType = "SPOT" - snapShot2.Pair = currency.NewPairFromString("LTCUSD") - - ws.Orderbook.LoadSnapshot(&snapShot2, "ExchangeTest", false) - - var snapShot3 orderbook.Base - asks = []orderbook.Item{ - {Price: 51, Amount: 1, ID: 1}, - {Price: 52, Amount: 0.5, ID: 2}, - {Price: 53, Amount: 2, ID: 3}, - {Price: 54, Amount: 3, ID: 4}, - {Price: 55, Amount: 5, ID: 5}, - {Price: 56, Amount: 2, ID: 6}, - {Price: 57, Amount: 1.5, ID: 7}, - {Price: 58, Amount: 0.5, ID: 8}, - {Price: 59, Amount: 23, ID: 9}, - {Price: 50, Amount: 9, ID: 10}, - {Price: 60, Amount: 7, ID: 11}, - } - - bids = []orderbook.Item{ - {Price: 49, Amount: 1, ID: 12}, - {Price: 48, Amount: 0.5, ID: 13}, - {Price: 47, Amount: 2, ID: 14}, - {Price: 46, Amount: 3, ID: 15}, - {Price: 45, Amount: 5, ID: 16}, - {Price: 44, Amount: 2, ID: 17}, - {Price: 43, Amount: 1.5, ID: 18}, - {Price: 42, Amount: 0.5, ID: 19}, - {Price: 41, Amount: 23, ID: 20}, - {Price: 40, Amount: 9, ID: 21}, - {Price: 39, Amount: 7, ID: 22}, - } - - snapShot3.Asks = asks - snapShot3.Bids = bids - snapShot3.AssetType = "FUTURES" - snapShot3.Pair = currency.NewPairFromString("LTCUSD") - - ws.Orderbook.LoadSnapshot(&snapShot3, "ExchangeTest", false) - - if len(ws.Orderbook.ob) != 3 { - t.Error("test failed - inserting orderbook data") - } -} - -func TestUpdate(t *testing.T) { - LTCUSDPAIR := currency.NewPairFromString("LTCUSD") - BTCUSDPAIR := currency.NewPairFromString("BTCUSD") - - bidTargets := []orderbook.Item{ - {Price: 49, Amount: 24}, // Amend - {Price: 48, Amount: 0}, // Delete - {Price: 1337, Amount: 100}, // Append - {Price: 1336, Amount: 0}, // Ghost delete - } - - askTargets := []orderbook.Item{ - {Price: 51, Amount: 24}, // Amend - {Price: 52, Amount: 0}, // Delete - {Price: 1337, Amount: 100}, // Append - {Price: 1336, Amount: 0}, // Ghost delete - } - err := ws.Orderbook.Update(bidTargets, - askTargets, - LTCUSDPAIR, - time.Now(), - "ExchangeTest", - "SPOT") - - if err != nil { - t.Error("test failed - OrderbookUpdate error", err) - } - - err = ws.Orderbook.Update(bidTargets, - askTargets, - LTCUSDPAIR, - time.Now(), - "ExchangeTest", - "FUTURES") - - if err != nil { - t.Error("test failed - OrderbookUpdate error", err) - } - - bidTargets = []orderbook.Item{ - {Price: 5999, Amount: 24}, // Amend - {Price: 5998, Amount: 0}, // Delete - {Price: 1337, Amount: 100}, // Append - {Price: 1336, Amount: 0}, // Ghost delete - } - - askTargets = []orderbook.Item{ - {Price: 6000, Amount: 24}, // Amend - {Price: 6001, Amount: 0}, // Delete - {Price: 1337, Amount: 100}, // Append - {Price: 1336, Amount: 0}, // Ghost delete - } - - err = ws.Orderbook.Update(bidTargets, - askTargets, - BTCUSDPAIR, - time.Now(), - "ExchangeTest", - "SPOT") - - if err != nil { - t.Error("test failed - OrderbookUpdate error", err) - } -} - func TestFunctionality(t *testing.T) { var w Websocket @@ -409,17 +227,6 @@ func TestUnsubscriptionWithExistingEntry(t *testing.T) { } } -// TestManageSubscriptionsWithoutFunctionality logic test -func TestManageSubscriptionsWithoutFunctionality(t *testing.T) { - w := Websocket{ - ShutdownC: make(chan struct{}, 1), - } - err := w.manageSubscriptions() - if err == nil { - t.Error("Requires functionality to work") - } -} - // TestManageSubscriptionsStartStop logic test func TestManageSubscriptionsStartStop(t *testing.T) { w := Websocket{ @@ -431,17 +238,17 @@ func TestManageSubscriptionsStartStop(t *testing.T) { close(w.ShutdownC) } -// TestWsConnectionMonitorNoConnection logic test -func TestWsConnectionMonitorNoConnection(t *testing.T) { +// TestConnectionMonitorNoConnection logic test +func TestConnectionMonitorNoConnection(t *testing.T) { w := Websocket{} w.DataHandler = make(chan interface{}, 1) w.ShutdownC = make(chan struct{}, 1) w.exchangeName = "hello" - go w.wsConnectionMonitor() + go w.connectionMonitor() err := <-w.DataHandler if !strings.EqualFold(err.(error).Error(), - fmt.Sprintf("%v WsConnectionMonitor: websocket disabled, shutting down", w.exchangeName)) { - t.Errorf("expecting error 'WsConnectionMonitor: websocket disabled, shutting down', received '%v'", err) + fmt.Sprintf("%v connectionMonitor: websocket disabled, shutting down", w.exchangeName)) { + t.Errorf("expecting error 'connectionMonitor: websocket disabled, shutting down', received '%v'", err) } } diff --git a/exchanges/wshandler/websocket_types.go b/exchanges/websocket/wshandler/wshandler_types.go similarity index 90% rename from exchanges/wshandler/websocket_types.go rename to exchanges/websocket/wshandler/wshandler_types.go index 3b1fc7d8..98d288b4 100644 --- a/exchanges/wshandler/websocket_types.go +++ b/exchanges/websocket/wshandler/wshandler_types.go @@ -4,8 +4,9 @@ import ( "sync" "time" + "github.com/gorilla/websocket" "github.com/thrasher-corp/gocryptotrader/currency" - "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook" ) // Websocket functionality list and state consts @@ -94,7 +95,7 @@ type Websocket struct { // ShutdownC is the main shutdown channel which controls all websocket go funcs ShutdownC chan struct{} // Orderbook is a local cache of orderbooks - Orderbook WebsocketOrderbookLocal + Orderbook wsorderbook.WebsocketOrderbookLocal // Wg defines a wait group for websocket routines for cleanly shutting down // routines Wg sync.WaitGroup @@ -113,14 +114,6 @@ type WebsocketChannelSubscription struct { Params map[string]interface{} } -// WebsocketOrderbookLocal defines a local cache of orderbooks for amending, -// appending and deleting changes and updates the main store in orderbook.go -type WebsocketOrderbookLocal struct { - ob []*orderbook.Base - lastUpdated time.Time - m sync.Mutex -} - // WebsocketResponse defines generalised data from the websocket connection type WebsocketResponse struct { Type int @@ -184,3 +177,20 @@ type WebsocketPositionUpdated struct { AssetType string Exchange string } + +// WebsocketConnection contains all the data needed to send a message to a WS +type WebsocketConnection struct { + sync.Mutex + Verbose bool + RateLimit float64 + ExchangeName string + URL string + ProxyURL string + Wg sync.WaitGroup + Connection *websocket.Conn + Shutdown chan struct{} + // These are the request IDs and the corresponding response JSON + IDResponses map[int64][]byte + ResponseCheckTimeout time.Duration + ResponseMaxLimit time.Duration +} diff --git a/exchanges/websocket/wsorderbook/wsorderbook.go b/exchanges/websocket/wsorderbook/wsorderbook.go new file mode 100644 index 00000000..80ed724e --- /dev/null +++ b/exchanges/websocket/wsorderbook/wsorderbook.go @@ -0,0 +1,242 @@ +package wsorderbook + +import ( + "fmt" + "sort" + "sync" + + "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" +) + +// Setup sets private variables +func (w *WebsocketOrderbookLocal) Setup(obBufferLimit int, bufferEnabled, sortBuffer, sortBufferByUpdateIDs, updateEntriesByID bool, exchangeName string) { + w.obBufferLimit = obBufferLimit + w.bufferEnabled = bufferEnabled + w.sortBuffer = sortBuffer + w.sortBufferByUpdateIDs = sortBufferByUpdateIDs + w.updateEntriesByID = updateEntriesByID + w.exchangeName = exchangeName +} + +// Update updates a local cache using bid targets and ask targets then updates +// main orderbook +// Volume == 0; deletion at price target +// Price target not found; append of price target +// Price target found; amend volume of price target +func (w *WebsocketOrderbookLocal) Update(orderbookUpdate *WebsocketOrderbookUpdate) error { + if (orderbookUpdate.Bids == nil && orderbookUpdate.Asks == nil) || + (len(orderbookUpdate.Bids) == 0 && len(orderbookUpdate.Asks) == 0) { + return fmt.Errorf("%v cannot have bids and ask targets both nil", w.exchangeName) + } + w.m.Lock() + defer w.m.Unlock() + if _, ok := w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType]; !ok { + return fmt.Errorf("ob.Base could not be found for Exchange %s CurrencyPair: %s AssetType: %s", + w.exchangeName, + orderbookUpdate.CurrencyPair.String(), + orderbookUpdate.AssetType) + } + if w.bufferEnabled { + overBufferLimit := w.processBufferUpdate(orderbookUpdate) + if !overBufferLimit { + return nil + } + } else { + w.processObUpdate(orderbookUpdate) + } + err := w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Process() + if err != nil { + return err + } + if w.bufferEnabled { + // Reset the buffer + w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType] = nil + } + return nil +} + +func (w *WebsocketOrderbookLocal) processBufferUpdate(orderbookUpdate *WebsocketOrderbookUpdate) bool { + if w.buffer == nil { + w.buffer = make(map[currency.Pair]map[string][]WebsocketOrderbookUpdate) + } + if w.buffer[orderbookUpdate.CurrencyPair] == nil { + w.buffer[orderbookUpdate.CurrencyPair] = make(map[string][]WebsocketOrderbookUpdate) + } + if len(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType]) <= w.obBufferLimit { + w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType] = append(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType], *orderbookUpdate) + if len(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType]) < w.obBufferLimit { + return false + } + } + if w.sortBuffer { + // sort by last updated to ensure each update is in order + if w.sortBufferByUpdateIDs { + sort.Slice(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType], func(i, j int) bool { + return w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][i].UpdateID < w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][j].UpdateID + }) + } else { + sort.Slice(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType], func(i, j int) bool { + return w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][i].UpdateTime.Before(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][j].UpdateTime) + }) + } + } + for i := 0; i < len(w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType]); i++ { + w.processObUpdate(&w.buffer[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType][i]) + } + return true +} + +func (w *WebsocketOrderbookLocal) processObUpdate(orderbookUpdate *WebsocketOrderbookUpdate) { + if w.updateEntriesByID { + w.updateByIDAndAction(orderbookUpdate) + } else { + var wg sync.WaitGroup + wg.Add(2) + go w.updateAsksByPrice(orderbookUpdate, &wg) + go w.updateBidsByPrice(orderbookUpdate, &wg) + wg.Wait() + } +} + +func (w *WebsocketOrderbookLocal) updateAsksByPrice(base *WebsocketOrderbookUpdate, wg *sync.WaitGroup) { + for j := 0; j < len(base.Asks); j++ { + found := false + for k := 0; k < len(w.ob[base.CurrencyPair][base.AssetType].Asks); k++ { + if w.ob[base.CurrencyPair][base.AssetType].Asks[k].Price == base.Asks[j].Price { + found = true + if base.Asks[j].Amount == 0 { + w.ob[base.CurrencyPair][base.AssetType].Asks = append(w.ob[base.CurrencyPair][base.AssetType].Asks[:k], + w.ob[base.CurrencyPair][base.AssetType].Asks[k+1:]...) + break + } + w.ob[base.CurrencyPair][base.AssetType].Asks[k].Amount = base.Asks[j].Amount + break + } + } + if !found { + w.ob[base.CurrencyPair][base.AssetType].Asks = append(w.ob[base.CurrencyPair][base.AssetType].Asks, base.Asks[j]) + } + } + sort.Slice(w.ob[base.CurrencyPair][base.AssetType].Asks, func(i, j int) bool { + return w.ob[base.CurrencyPair][base.AssetType].Asks[i].Price < w.ob[base.CurrencyPair][base.AssetType].Asks[j].Price + }) + wg.Done() +} + +func (w *WebsocketOrderbookLocal) updateBidsByPrice(base *WebsocketOrderbookUpdate, wg *sync.WaitGroup) { + for j := 0; j < len(base.Bids); j++ { + found := false + for k := 0; k < len(w.ob[base.CurrencyPair][base.AssetType].Bids); k++ { + if w.ob[base.CurrencyPair][base.AssetType].Bids[k].Price == base.Bids[j].Price { + found = true + if base.Bids[j].Amount == 0 { + w.ob[base.CurrencyPair][base.AssetType].Bids = append(w.ob[base.CurrencyPair][base.AssetType].Bids[:k], + w.ob[base.CurrencyPair][base.AssetType].Bids[k+1:]...) + break + } + w.ob[base.CurrencyPair][base.AssetType].Bids[k].Amount = base.Bids[j].Amount + break + } + } + if !found { + w.ob[base.CurrencyPair][base.AssetType].Bids = append(w.ob[base.CurrencyPair][base.AssetType].Bids, base.Bids[j]) + } + } + sort.Slice(w.ob[base.CurrencyPair][base.AssetType].Bids, func(i, j int) bool { + return w.ob[base.CurrencyPair][base.AssetType].Bids[i].Price > w.ob[base.CurrencyPair][base.AssetType].Bids[j].Price + }) + wg.Done() +} + +// updateByIDAndAction will receive an action to execute against the orderbook +// it will then match by IDs instead of price to perform the action +func (w *WebsocketOrderbookLocal) updateByIDAndAction(orderbookUpdate *WebsocketOrderbookUpdate) { + switch orderbookUpdate.Action { + case "update": + for _, target := range orderbookUpdate.Bids { + for i := range w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids { + if w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[i].ID == target.ID { + w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[i].Amount = target.Amount + break + } + } + } + for _, target := range orderbookUpdate.Asks { + for i := range w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks { + if w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[i].ID == target.ID { + w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[i].Amount = target.Amount + break + } + } + } + case "delete": + for _, target := range orderbookUpdate.Bids { + for i := 0; i < len(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids); i++ { + if w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[i].ID == target.ID { + w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids = append(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[:i], + w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids[i+1:]...) + i-- + break + } + } + } + for _, target := range orderbookUpdate.Asks { + for i := 0; i < len(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks); i++ { + if w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[i].ID == target.ID { + w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks = append(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[:i], + w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks[i+1:]...) + i-- + break + } + } + } + case "insert": + w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids = append(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Bids, orderbookUpdate.Bids...) + w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks = append(w.ob[orderbookUpdate.CurrencyPair][orderbookUpdate.AssetType].Asks, orderbookUpdate.Asks...) + } +} + +// LoadSnapshot loads initial snapshot of ob data, overwrite allows full +// ob to be completely rewritten because the exchange is a doing a full +// update not an incremental one +func (w *WebsocketOrderbookLocal) LoadSnapshot(newOrderbook *orderbook.Base, overwrite bool) error { + if len(newOrderbook.Asks) == 0 || len(newOrderbook.Bids) == 0 { + return fmt.Errorf("%v snapshot ask and bids are nil", w.exchangeName) + } + w.m.Lock() + defer w.m.Unlock() + if w.ob == nil { + w.ob = make(map[currency.Pair]map[string]*orderbook.Base) + } + if w.ob[newOrderbook.Pair] == nil { + w.ob[newOrderbook.Pair] = make(map[string]*orderbook.Base) + } + if w.ob[newOrderbook.Pair][newOrderbook.AssetType] != nil && + (len(w.ob[newOrderbook.Pair][newOrderbook.AssetType].Asks) > 0 || + len(w.ob[newOrderbook.Pair][newOrderbook.AssetType].Bids) > 0) { + if overwrite { + w.ob[newOrderbook.Pair][newOrderbook.AssetType] = newOrderbook + return newOrderbook.Process() + } + return fmt.Errorf("%v snapshot instance already found", w.exchangeName) + } + w.ob[newOrderbook.Pair][newOrderbook.AssetType] = newOrderbook + return newOrderbook.Process() +} + +// GetOrderbook use sparingly. Modifying anything here will ruin hash calculation and cause problems +func (w *WebsocketOrderbookLocal) GetOrderbook(p currency.Pair, assetType string) *orderbook.Base { + w.m.Lock() + defer w.m.Unlock() + return w.ob[p][assetType] +} + +// FlushCache flushes w.ob data to be garbage collected and refreshed when a +// connection is lost and reconnected +func (w *WebsocketOrderbookLocal) FlushCache() { + w.m.Lock() + w.ob = nil + w.buffer = nil + w.m.Unlock() +} diff --git a/exchanges/websocket/wsorderbook/wsorderbook_test.go b/exchanges/websocket/wsorderbook/wsorderbook_test.go new file mode 100644 index 00000000..6f4ddfef --- /dev/null +++ b/exchanges/websocket/wsorderbook/wsorderbook_test.go @@ -0,0 +1,582 @@ +package wsorderbook + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" +) + +var itemArray = [][]orderbook.Item{ + {{Price: 1000, Amount: 1, ID: 1}}, + {{Price: 2000, Amount: 1, ID: 2}}, + {{Price: 3000, Amount: 1, ID: 3}}, + {{Price: 3000, Amount: 2, ID: 4}}, + {{Price: 4000, Amount: 0, ID: 6}}, + {{Price: 5000, Amount: 1, ID: 5}}, +} + +const ( + exchangeName = "exchangeTest" + spot = orderbook.Spot +) + +func createSnapshot() (obl *WebsocketOrderbookLocal, curr currency.Pair, asks, bids []orderbook.Item, err error) { + var snapShot1 orderbook.Base + curr = currency.NewPairFromString("BTCUSD") + asks = []orderbook.Item{ + {Price: 4000, Amount: 1, ID: 6}, + } + bids = []orderbook.Item{ + {Price: 4000, Amount: 1, ID: 6}, + } + snapShot1.Asks = asks + snapShot1.Bids = bids + snapShot1.AssetType = spot + snapShot1.Pair = curr + obl = &WebsocketOrderbookLocal{} + err = obl.LoadSnapshot(&snapShot1, false) + return +} + +// BenchmarkBufferPerformance demonstrates buffer more performant than multi process calls +func BenchmarkBufferPerformance(b *testing.B) { + obl, curr, asks, bids, err := createSnapshot() + if err != nil { + b.Fatal(err) + } + obl.exchangeName = exchangeName + obl.sortBuffer = true + update := &WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + } + for i := 0; i < b.N; i++ { + randomIndex := rand.Intn(5) + update.Asks = itemArray[randomIndex] + update.Bids = itemArray[randomIndex] + err = obl.Update(update) + if err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkBufferSortingPerformance benchmark +func BenchmarkBufferSortingPerformance(b *testing.B) { + obl, curr, asks, bids, err := createSnapshot() + if err != nil { + b.Fatal(err) + } + obl.exchangeName = exchangeName + obl.sortBuffer = true + obl.bufferEnabled = true + obl.obBufferLimit = 5 + update := &WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + } + for i := 0; i < b.N; i++ { + randomIndex := rand.Intn(5) + update.Asks = itemArray[randomIndex] + update.Bids = itemArray[randomIndex] + err = obl.Update(update) + if err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkNoBufferPerformance demonstrates orderbook process less performant than buffer +func BenchmarkNoBufferPerformance(b *testing.B) { + obl, curr, asks, bids, err := createSnapshot() + if err != nil { + b.Fatal(err) + } + obl.exchangeName = exchangeName + update := &WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + } + for i := 0; i < b.N; i++ { + randomIndex := rand.Intn(5) + update.Asks = itemArray[randomIndex] + update.Bids = itemArray[randomIndex] + err = obl.Update(update) + if err != nil { + b.Fatal(err) + } + } +} + +// TestHittingTheBuffer logic test +func TestHittingTheBuffer(t *testing.T) { + obl, curr, _, _, err := createSnapshot() + if err != nil { + t.Fatal(err) + } + obl.exchangeName = exchangeName + obl.bufferEnabled = true + obl.obBufferLimit = 5 + for i := 0; i < len(itemArray); i++ { + asks := itemArray[i] + bids := itemArray[i] + err = obl.Update(&WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + }) + if err != nil { + t.Fatal(err) + } + } + if len(obl.ob[curr][spot].Asks) != 3 { + t.Log(obl.ob[curr][spot]) + t.Errorf("expected 3 entries, received: %v", len(obl.ob[curr][spot].Asks)) + } + if len(obl.ob[curr][spot].Bids) != 3 { + t.Errorf("expected 3 entries, received: %v", len(obl.ob[curr][spot].Bids)) + } +} + +// TestInsertWithIDs logic test +func TestInsertWithIDs(t *testing.T) { + obl, curr, _, _, err := createSnapshot() + if err != nil { + t.Fatal(err) + } + obl.exchangeName = exchangeName + obl.bufferEnabled = true + obl.updateEntriesByID = true + obl.obBufferLimit = 5 + for i := 0; i < len(itemArray); i++ { + asks := itemArray[i] + bids := itemArray[i] + err = obl.Update(&WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + Action: "insert", + }) + if err != nil { + t.Fatal(err) + } + } + if len(obl.ob[curr][spot].Asks) != 6 { + t.Errorf("expected 6 entries, received: %v", len(obl.ob[curr][spot].Asks)) + } + if len(obl.ob[curr][spot].Bids) != 6 { + t.Errorf("expected 6 entries, received: %v", len(obl.ob[curr][spot].Bids)) + } +} + +// TestSortIDs logic test +func TestSortIDs(t *testing.T) { + obl, curr, _, _, err := createSnapshot() + if err != nil { + t.Fatal(err) + } + obl.exchangeName = exchangeName + obl.bufferEnabled = true + obl.sortBufferByUpdateIDs = true + obl.sortBuffer = true + obl.obBufferLimit = 5 + for i := 0; i < len(itemArray); i++ { + asks := itemArray[i] + bids := itemArray[i] + err = obl.Update(&WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateID: int64(i), + AssetType: spot, + }) + if err != nil { + t.Fatal(err) + } + } + if len(obl.ob[curr][spot].Asks) != 3 { + t.Errorf("expected 6 entries, received: %v", len(obl.ob[curr][spot].Asks)) + } + if len(obl.ob[curr][spot].Bids) != 3 { + t.Errorf("expected 6 entries, received: %v", len(obl.ob[curr][spot].Bids)) + } +} + +// TestDeleteWithIDs logic test +func TestDeleteWithIDs(t *testing.T) { + obl, curr, _, _, err := createSnapshot() + if err != nil { + t.Fatal(err) + } + obl.exchangeName = exchangeName + obl.updateEntriesByID = true + for i := 0; i < len(itemArray); i++ { + asks := itemArray[i] + bids := itemArray[i] + err = obl.Update(&WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + Action: "delete", + }) + if err != nil { + t.Fatal(err) + } + } + if len(obl.ob[curr][spot].Asks) != 0 { + t.Errorf("expected 0 entries, received: %v", len(obl.ob[curr][spot].Asks)) + } + if len(obl.ob[curr][spot].Bids) != 0 { + t.Errorf("expected 0 entries, received: %v", len(obl.ob[curr][spot].Bids)) + } +} + +// TestUpdateWithIDs logic test +func TestUpdateWithIDs(t *testing.T) { + obl, curr, _, _, err := createSnapshot() + if err != nil { + t.Fatal(err) + } + obl.exchangeName = exchangeName + obl.updateEntriesByID = true + for i := 0; i < len(itemArray); i++ { + asks := itemArray[i] + bids := itemArray[i] + err = obl.Update(&WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + Action: "update", + }) + if err != nil { + t.Fatal(err) + } + } + if len(obl.ob[curr][spot].Asks) != 1 { + t.Log(obl.ob[curr][spot]) + t.Errorf("expected 1 entries, received: %v", len(obl.ob[curr][spot].Asks)) + } + if len(obl.ob[curr][spot].Bids) != 1 { + t.Errorf("expected 1 entries, received: %v", len(obl.ob[curr][spot].Bids)) + } +} + +// TestOutOfOrderIDs logic test +func TestOutOfOrderIDs(t *testing.T) { + obl, curr, _, _, err := createSnapshot() + if err != nil { + t.Fatal(err) + } + outOFOrderIDs := []int64{2, 1, 5, 3, 4, 6} + if itemArray[0][0].Price != 1000 { + t.Errorf("expected sorted price to be 3000, received: %v", itemArray[1][0].Price) + } + obl.exchangeName = exchangeName + obl.bufferEnabled = true + obl.sortBuffer = true + obl.obBufferLimit = 5 + for i := 0; i < len(itemArray); i++ { + asks := itemArray[i] + err = obl.Update(&WebsocketOrderbookUpdate{ + Asks: asks, + CurrencyPair: curr, + UpdateID: outOFOrderIDs[i], + AssetType: spot, + }) + if err != nil { + t.Fatal(err) + } + } + // Index 1 since index 0 is price 7000 + if obl.ob[curr][spot].Asks[1].Price != 2000 { + t.Errorf("expected sorted price to be 3000, received: %v", obl.ob[curr][spot].Asks[1].Price) + } +} + +// TestRunUpdateWithoutSnapshot logic test +func TestRunUpdateWithoutSnapshot(t *testing.T) { + var obl WebsocketOrderbookLocal + var snapShot1 orderbook.Base + curr := currency.NewPairFromString("BTCUSD") + asks := []orderbook.Item{ + {Price: 4000, Amount: 1, ID: 8}, + } + bids := []orderbook.Item{ + {Price: 5999, Amount: 1, ID: 8}, + {Price: 4000, Amount: 1, ID: 9}, + } + snapShot1.Asks = asks + snapShot1.Bids = bids + snapShot1.AssetType = spot + snapShot1.Pair = curr + obl.exchangeName = exchangeName + err := obl.Update(&WebsocketOrderbookUpdate{ + Bids: bids, + Asks: asks, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + }) + if err == nil { + t.Fatal("expected an error running update with no snapshot loaded") + } + if err.Error() != "ob.Base could not be found for Exchange exchangeTest CurrencyPair: BTCUSD AssetType: SPOT" { + t.Fatal(err) + } +} + +// TestRunUpdateWithoutAnyUpdates logic test +func TestRunUpdateWithoutAnyUpdates(t *testing.T) { + var obl WebsocketOrderbookLocal + var snapShot1 orderbook.Base + curr := currency.NewPairFromString("BTCUSD") + snapShot1.Asks = []orderbook.Item{} + snapShot1.Bids = []orderbook.Item{} + snapShot1.AssetType = spot + snapShot1.Pair = curr + obl.exchangeName = exchangeName + err := obl.Update(&WebsocketOrderbookUpdate{ + Bids: snapShot1.Asks, + Asks: snapShot1.Bids, + CurrencyPair: curr, + UpdateTime: time.Now(), + AssetType: spot, + }) + if err == nil { + t.Fatal("expected an error running update with no snapshot loaded") + } + if err.Error() != fmt.Sprintf("%v cannot have bids and ask targets both nil", exchangeName) { + t.Fatal("expected nil asks and bids error") + } +} + +// TestRunSnapshotWithNoData logic test +func TestRunSnapshotWithNoData(t *testing.T) { + var obl WebsocketOrderbookLocal + var snapShot1 orderbook.Base + curr := currency.NewPairFromString("BTCUSD") + snapShot1.Asks = []orderbook.Item{} + snapShot1.Bids = []orderbook.Item{} + snapShot1.AssetType = spot + snapShot1.Pair = curr + snapShot1.ExchangeName = "test" + obl.exchangeName = "test" + err := obl.LoadSnapshot(&snapShot1, + false) + if err == nil { + t.Fatal("expected an error loading a snapshot") + } + if err.Error() != "test snapshot ask and bids are nil" { + t.Fatal(err) + } +} + +// TestLoadSnapshotWithOverride logic test +func TestLoadSnapshotWithOverride(t *testing.T) { + var obl WebsocketOrderbookLocal + var snapShot1 orderbook.Base + curr := currency.NewPairFromString("BTCUSD") + asks := []orderbook.Item{ + {Price: 4000, Amount: 1, ID: 8}, + } + bids := []orderbook.Item{ + {Price: 4000, Amount: 1, ID: 9}, + } + snapShot1.Asks = asks + snapShot1.Bids = bids + snapShot1.AssetType = spot + snapShot1.Pair = curr + err := obl.LoadSnapshot(&snapShot1, false) + if err != nil { + t.Error(err) + } + err = obl.LoadSnapshot(&snapShot1, false) + if err == nil { + t.Error("expected error: 'snapshot instance already found'") + } + err = obl.LoadSnapshot(&snapShot1, true) + if err != nil { + t.Error(err) + } +} + +// TestInsertWithIDs logic test +func TestFlushCache(t *testing.T) { + obl, curr, _, _, err := createSnapshot() + if err != nil { + t.Fatal(err) + } + if obl.ob[curr][spot] == nil { + t.Error("expected ob to have ask entries") + } + obl.FlushCache() + if obl.ob[curr][spot] != nil { + t.Error("expected ob be flushed") + } + +} + +// TestInsertingSnapShots logic test +func TestInsertingSnapShots(t *testing.T) { + var obl WebsocketOrderbookLocal + var snapShot1 orderbook.Base + asks := []orderbook.Item{ + {Price: 6000, Amount: 1, ID: 1}, + {Price: 6001, Amount: 0.5, ID: 2}, + {Price: 6002, Amount: 2, ID: 3}, + {Price: 6003, Amount: 3, ID: 4}, + {Price: 6004, Amount: 5, ID: 5}, + {Price: 6005, Amount: 2, ID: 6}, + {Price: 6006, Amount: 1.5, ID: 7}, + {Price: 6007, Amount: 0.5, ID: 8}, + {Price: 6008, Amount: 23, ID: 9}, + {Price: 6009, Amount: 9, ID: 10}, + {Price: 6010, Amount: 7, ID: 11}, + } + + bids := []orderbook.Item{ + {Price: 5999, Amount: 1, ID: 12}, + {Price: 5998, Amount: 0.5, ID: 13}, + {Price: 5997, Amount: 2, ID: 14}, + {Price: 5996, Amount: 3, ID: 15}, + {Price: 5995, Amount: 5, ID: 16}, + {Price: 5994, Amount: 2, ID: 17}, + {Price: 5993, Amount: 1.5, ID: 18}, + {Price: 5992, Amount: 0.5, ID: 19}, + {Price: 5991, Amount: 23, ID: 20}, + {Price: 5990, Amount: 9, ID: 21}, + {Price: 5989, Amount: 7, ID: 22}, + } + + snapShot1.Asks = asks + snapShot1.Bids = bids + snapShot1.AssetType = spot + snapShot1.Pair = currency.NewPairFromString("BTCUSD") + err := obl.LoadSnapshot(&snapShot1, false) + if err != nil { + t.Fatal(err) + } + var snapShot2 orderbook.Base + asks = []orderbook.Item{ + {Price: 51, Amount: 1, ID: 1}, + {Price: 52, Amount: 0.5, ID: 2}, + {Price: 53, Amount: 2, ID: 3}, + {Price: 54, Amount: 3, ID: 4}, + {Price: 55, Amount: 5, ID: 5}, + {Price: 56, Amount: 2, ID: 6}, + {Price: 57, Amount: 1.5, ID: 7}, + {Price: 58, Amount: 0.5, ID: 8}, + {Price: 59, Amount: 23, ID: 9}, + {Price: 50, Amount: 9, ID: 10}, + {Price: 60, Amount: 7, ID: 11}, + } + + bids = []orderbook.Item{ + {Price: 49, Amount: 1, ID: 12}, + {Price: 48, Amount: 0.5, ID: 13}, + {Price: 47, Amount: 2, ID: 14}, + {Price: 46, Amount: 3, ID: 15}, + {Price: 45, Amount: 5, ID: 16}, + {Price: 44, Amount: 2, ID: 17}, + {Price: 43, Amount: 1.5, ID: 18}, + {Price: 42, Amount: 0.5, ID: 19}, + {Price: 41, Amount: 23, ID: 20}, + {Price: 40, Amount: 9, ID: 21}, + {Price: 39, Amount: 7, ID: 22}, + } + + snapShot2.Asks = asks + snapShot2.Bids = bids + snapShot2.AssetType = spot + snapShot2.Pair = currency.NewPairFromString("LTCUSD") + err = obl.LoadSnapshot(&snapShot2, false) + if err != nil { + t.Fatal(err) + } + var snapShot3 orderbook.Base + asks = []orderbook.Item{ + {Price: 511, Amount: 1, ID: 1}, + {Price: 52, Amount: 0.5, ID: 2}, + {Price: 53, Amount: 2, ID: 3}, + {Price: 54, Amount: 3, ID: 4}, + {Price: 55, Amount: 5, ID: 5}, + {Price: 56, Amount: 2, ID: 6}, + {Price: 57, Amount: 1.5, ID: 7}, + {Price: 58, Amount: 0.5, ID: 8}, + {Price: 59, Amount: 23, ID: 9}, + {Price: 50, Amount: 9, ID: 10}, + {Price: 60, Amount: 7, ID: 11}, + } + + bids = []orderbook.Item{ + {Price: 49, Amount: 1, ID: 12}, + {Price: 48, Amount: 0.5, ID: 13}, + {Price: 47, Amount: 2, ID: 14}, + {Price: 46, Amount: 3, ID: 15}, + {Price: 45, Amount: 5, ID: 16}, + {Price: 44, Amount: 2, ID: 17}, + {Price: 43, Amount: 1.5, ID: 18}, + {Price: 42, Amount: 0.5, ID: 19}, + {Price: 41, Amount: 23, ID: 20}, + {Price: 40, Amount: 9, ID: 21}, + {Price: 39, Amount: 7, ID: 22}, + } + + snapShot3.Asks = asks + snapShot3.Bids = bids + snapShot3.AssetType = "FUTURES" + snapShot3.Pair = currency.NewPairFromString("LTCUSD") + err = obl.LoadSnapshot(&snapShot3, false) + if err != nil { + t.Fatal(err) + } + if obl.ob[snapShot1.Pair][snapShot1.AssetType].Asks[0] != snapShot1.Asks[0] { + t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot1.Asks[0], obl.ob[snapShot1.Pair][snapShot1.AssetType].Asks[0]) + } + if obl.ob[snapShot2.Pair][snapShot2.AssetType].Asks[0] != snapShot2.Asks[0] { + t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot2.Asks[0], obl.ob[snapShot2.Pair][snapShot2.AssetType].Asks[0]) + } + if obl.ob[snapShot3.Pair][snapShot3.AssetType].Asks[0] != snapShot3.Asks[0] { + t.Errorf("loaded data mismatch. Expected %v, received %v", snapShot3.Asks[0], obl.ob[snapShot3.Pair][snapShot3.AssetType].Asks[0]) + } +} + +func TestGetOrderbook(t *testing.T) { + obl, curr, _, _, err := createSnapshot() + if err != nil { + t.Fatal(err) + } + ob := obl.GetOrderbook(curr, spot) + if obl.ob[curr][spot] != ob { + t.Error("Failed to get orderbook") + } +} + +func TestSetup(t *testing.T) { + w := WebsocketOrderbookLocal{} + w.Setup(1, true, true, true, true, "hi") + if w.obBufferLimit != 1 || !w.bufferEnabled || !w.sortBuffer || !w.sortBufferByUpdateIDs || !w.updateEntriesByID || w.exchangeName != "hi" { + t.Errorf("Setup incorrectly loaded %v", w) + } +} diff --git a/exchanges/websocket/wsorderbook/wsorderbook_types.go b/exchanges/websocket/wsorderbook/wsorderbook_types.go new file mode 100644 index 00000000..07d7fc04 --- /dev/null +++ b/exchanges/websocket/wsorderbook/wsorderbook_types.go @@ -0,0 +1,34 @@ +package wsorderbook + +import ( + "sync" + "time" + + "github.com/thrasher-corp/gocryptotrader/currency" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" +) + +// WebsocketOrderbookLocal defines a local cache of orderbooks for amending, +// appending and deleting changes and updates the main store in wsorderbook.go +type WebsocketOrderbookLocal struct { + ob map[currency.Pair]map[string]*orderbook.Base + buffer map[currency.Pair]map[string][]WebsocketOrderbookUpdate + obBufferLimit int + bufferEnabled bool + sortBuffer bool + sortBufferByUpdateIDs bool // When timestamps aren't provided, an id can help sort + updateEntriesByID bool // Use the update IDs to match ob entries + exchangeName string + m sync.Mutex +} + +// WebsocketOrderbookUpdate stores orderbook updates and dictates what features to use when processing +type WebsocketOrderbookUpdate struct { + UpdateID int64 // Used when no time is provided + UpdateTime time.Time + AssetType string + Action string // Used in conjunction with UpdateEntriesByID + Bids []orderbook.Item + Asks []orderbook.Item + CurrencyPair currency.Pair +} diff --git a/exchanges/wshandler/websocket_connection.go b/exchanges/wshandler/websocket_connection.go deleted file mode 100644 index 6bcf41f1..00000000 --- a/exchanges/wshandler/websocket_connection.go +++ /dev/null @@ -1,177 +0,0 @@ -package wshandler - -import ( - "bytes" - "compress/flate" - "compress/gzip" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "sync" - "time" - - "github.com/gorilla/websocket" - "github.com/thrasher-corp/gocryptotrader/common" - log "github.com/thrasher-corp/gocryptotrader/logger" -) - -// AddResponseWithID adds data to IDResponses with locks and a nil check -func (w *WebsocketConnection) AddResponseWithID(id int64, data []byte) { - w.Lock() - defer w.Unlock() - if w.IDResponses == nil { - w.IDResponses = make(map[int64][]byte) - } - w.IDResponses[id] = data -} - -// Dial sets proxy urls and then connects to the websocket -func (w *WebsocketConnection) Dial(dialer *websocket.Dialer, headers http.Header) error { - if w.ProxyURL != "" { - proxy, err := url.Parse(w.ProxyURL) - if err != nil { - return err - } - dialer.Proxy = http.ProxyURL(proxy) - } - - var err error - var conStatus *http.Response - w.Connection, conStatus, err = dialer.Dial(w.URL, headers) - if err != nil { - if conStatus != nil { - return fmt.Errorf("%v %v %v Error: %v", w.URL, conStatus, conStatus.StatusCode, err) - } - return fmt.Errorf("%v Error: %v", w.URL, err) - } - return nil -} - -// SendMessage the one true message request. Sends message to WS -func (w *WebsocketConnection) SendMessage(data interface{}) error { - w.Lock() - defer w.Unlock() - json, err := common.JSONEncode(data) - if err != nil { - return err - } - if w.Verbose { - log.Debugf("%v sending message to websocket %v", w.ExchangeName, string(json)) - } - if w.RateLimit > 0 { - time.Sleep(time.Duration(w.RateLimit) * time.Millisecond) - } - return w.Connection.WriteMessage(websocket.TextMessage, json) -} - -// SendMessageReturnResponse will send a WS message to the connection -// It will then run a goroutine to await a JSON response -// If there is no response it will return an error -func (w *WebsocketConnection) SendMessageReturnResponse(id int64, request interface{}) ([]byte, error) { - err := w.SendMessage(request) - if err != nil { - return nil, err - } - var wg sync.WaitGroup - wg.Add(1) - go w.WaitForResult(id, &wg) - defer func() { - delete(w.IDResponses, id) - }() - wg.Wait() - if _, ok := w.IDResponses[id]; !ok { - return nil, fmt.Errorf("timeout waiting for response with ID %v", id) - } - - return w.IDResponses[id], nil -} - -// WaitForResult will keep checking w.IDResponses for a response ID -// If the timer expires, it will return without -func (w *WebsocketConnection) WaitForResult(id int64, wg *sync.WaitGroup) { - defer wg.Done() - timer := time.NewTimer(w.ResponseMaxLimit) - for { - select { - case <-timer.C: - return - default: - w.Lock() - for k := range w.IDResponses { - if k == id { - w.Unlock() - return - } - } - w.Unlock() - time.Sleep(w.ResponseCheckTimeout) - } - } -} - -// ReadMessage reads messages, can handle text, gzip and binary -func (w *WebsocketConnection) ReadMessage() (WebsocketResponse, error) { - mType, resp, err := w.Connection.ReadMessage() - if err != nil { - return WebsocketResponse{}, err - } - var standardMessage []byte - switch mType { - case websocket.TextMessage: - standardMessage = resp - case websocket.BinaryMessage: - standardMessage, err = w.parseBinaryResponse(resp) - if err != nil { - return WebsocketResponse{}, err - } - } - if w.Verbose { - log.Debugf("%v Websocket message received: %v", - w.ExchangeName, - string(standardMessage)) - } - return WebsocketResponse{Raw: standardMessage, Type: mType}, nil -} - -// parseBinaryResponse parses a websocket binaray response into a usable byte array -func (w *WebsocketConnection) parseBinaryResponse(resp []byte) ([]byte, error) { - var standardMessage []byte - var err error - // Detect GZIP - if resp[0] == 31 && resp[1] == 139 { - b := bytes.NewReader(resp) - var gReader *gzip.Reader - gReader, err = gzip.NewReader(b) - if err != nil { - return standardMessage, err - } - standardMessage, err = ioutil.ReadAll(gReader) - if err != nil { - return standardMessage, err - } - err = gReader.Close() - if err != nil { - return standardMessage, err - } - } else { - reader := flate.NewReader(bytes.NewReader(resp)) - standardMessage, err = ioutil.ReadAll(reader) - if err != nil { - return standardMessage, err - } - err = reader.Close() - if err != nil { - return standardMessage, err - } - } - return standardMessage, nil -} - -// GenerateMessageID Creates a messageID to checkout -func (w *WebsocketConnection) GenerateMessageID(useNano bool) int64 { - if useNano { - return time.Now().UnixNano() - } - return time.Now().Unix() -} diff --git a/exchanges/wshandler/websocket_connection_test.go b/exchanges/wshandler/websocket_connection_test.go deleted file mode 100644 index e8bf13db..00000000 --- a/exchanges/wshandler/websocket_connection_test.go +++ /dev/null @@ -1,203 +0,0 @@ -package wshandler - -import ( - "bytes" - "compress/flate" - "compress/gzip" - "errors" - "net/http" - "os" - "strings" - "testing" - "time" - - "github.com/gorilla/websocket" - "github.com/thrasher-corp/gocryptotrader/common" - "github.com/thrasher-corp/gocryptotrader/currency" -) - -const ( - websocketTestURL = "wss://www.bitmex.com/realtime" - returnResponseURL = "wss://ws.kraken.com" - useProxyTests = false // Disabled by default. Freely available proxy servers that work all the time are difficult to find - proxyURL = "http://212.186.171.4:80" // Replace with a usable proxy server -) - -var wc *WebsocketConnection -var dialer websocket.Dialer - -type testStruct struct { - Error error - WC WebsocketConnection -} - -type testRequest struct { - Event string `json:"event"` - RequestID int64 `json:"reqid,omitempty"` - Pairs []string `json:"pair"` - Subscription testRequestData `json:"subscription,omitempty"` -} - -// testRequestData contains details on WS channel -type testRequestData struct { - Name string `json:"name,omitempty"` - Interval int64 `json:"interval,omitempty"` - Depth int64 `json:"depth,omitempty"` -} - -type testResponse struct { - RequestID int64 `json:"reqid,omitempty"` -} - -// TestMain setup test -func TestMain(m *testing.M) { - wc = &WebsocketConnection{ - ExchangeName: "test", - Verbose: true, - URL: returnResponseURL, - ResponseMaxLimit: 7000000000, - ResponseCheckTimeout: 30000000, - } - os.Exit(m.Run()) -} - -// TestDial logic test -func TestDial(t *testing.T) { - var testCases = []testStruct{ - {Error: nil, WC: WebsocketConnection{ExchangeName: "test1", Verbose: true, URL: websocketTestURL, RateLimit: 10, ResponseCheckTimeout: 30000000, ResponseMaxLimit: 7000000000}}, - {Error: errors.New(" Error: malformed ws or wss URL"), WC: WebsocketConnection{ExchangeName: "test2", Verbose: true, URL: "", ResponseCheckTimeout: 30000000, ResponseMaxLimit: 7000000000}}, - {Error: nil, WC: WebsocketConnection{ExchangeName: "test3", Verbose: true, URL: websocketTestURL, ProxyURL: proxyURL, ResponseCheckTimeout: 30000000, ResponseMaxLimit: 7000000000}}, - } - for i := 0; i < len(testCases); i++ { - testData := &testCases[i] - t.Run(testData.WC.ExchangeName, func(t *testing.T) { - if testData.WC.ProxyURL != "" && !useProxyTests { - t.Skip("Proxy testing not enabled, skipping") - } - err := testData.WC.Dial(&dialer, http.Header{}) - if err != nil { - if testData.Error != nil && err.Error() == testData.Error.Error() { - return - } - t.Fatal(err) - } - }) - } -} - -// TestSendMessage logic test -func TestSendMessage(t *testing.T) { - var testCases = []testStruct{ - {Error: nil, WC: WebsocketConnection{ExchangeName: "test1", Verbose: true, URL: websocketTestURL, RateLimit: 10, ResponseCheckTimeout: 30000000, ResponseMaxLimit: 7000000000}}, - {Error: errors.New(" Error: malformed ws or wss URL"), WC: WebsocketConnection{ExchangeName: "test2", Verbose: true, URL: "", ResponseCheckTimeout: 30000000, ResponseMaxLimit: 7000000000}}, - {Error: nil, WC: WebsocketConnection{ExchangeName: "test3", Verbose: true, URL: websocketTestURL, ProxyURL: proxyURL, ResponseCheckTimeout: 30000000, ResponseMaxLimit: 7000000000}}, - } - for i := 0; i < len(testCases); i++ { - testData := &testCases[i] - t.Run(testData.WC.ExchangeName, func(t *testing.T) { - if testData.WC.ProxyURL != "" && !useProxyTests { - t.Skip("Proxy testing not enabled, skipping") - } - err := testData.WC.Dial(&dialer, http.Header{}) - if err != nil { - if testData.Error != nil && err.Error() == testData.Error.Error() { - return - } - t.Fatal(err) - } - err = testData.WC.SendMessage("ping") - if err != nil { - t.Error(err) - } - }) - } -} - -// TestSendMessageWithResponse logic test -func TestSendMessageWithResponse(t *testing.T) { - if wc.ProxyURL != "" && !useProxyTests { - t.Skip("Proxy testing not enabled, skipping") - } - err := wc.Dial(&dialer, http.Header{}) - if err != nil { - t.Fatal(err) - } - go readMesages(wc, t) - - request := testRequest{ - Event: "subscribe", - Pairs: []string{currency.NewPairWithDelimiter("XBT", "USD", "/").String()}, - Subscription: testRequestData{ - Name: "ticker", - }, - RequestID: wc.GenerateMessageID(true), - } - _, err = wc.SendMessageReturnResponse(request.RequestID, request) - if err != nil { - t.Error(err) - } -} - -// TestParseBinaryResponse logic test -func TestParseBinaryResponse(t *testing.T) { - var b bytes.Buffer - w := gzip.NewWriter(&b) - w.Write([]byte("hello")) - w.Close() - resp, err := wc.parseBinaryResponse(b.Bytes()) - if err != nil { - t.Error(err) - } - if !strings.EqualFold(string(resp), "hello") { - t.Errorf("GZip conversion failed. Received: '%v', Expected: 'hello'", string(resp)) - } - - var b2 bytes.Buffer - w2, err2 := flate.NewWriter(&b2, 1) - if err2 != nil { - t.Error(err2) - } - w2.Write([]byte("hello")) - w2.Close() - resp2, err3 := wc.parseBinaryResponse(b2.Bytes()) - if err3 != nil { - t.Error(err3) - } - if !strings.EqualFold(string(resp2), "hello") { - t.Errorf("GZip conversion failed. Received: '%v', Expected: 'hello'", string(resp2)) - } -} - -// TestAddResponseWithID logic test -func TestAddResponseWithID(t *testing.T) { - wc.IDResponses = nil - wc.AddResponseWithID(0, []byte("hi")) - wc.AddResponseWithID(1, []byte("hi")) -} - -// readMesages helper func -func readMesages(wc *WebsocketConnection, t *testing.T) { - timer := time.NewTimer(20 * time.Second) - for { - select { - case <-timer.C: - return - default: - resp, err := wc.ReadMessage() - if err != nil { - t.Error(err) - return - } - var incoming testResponse - err = common.JSONDecode(resp.Raw, &incoming) - if err != nil { - t.Error(err) - return - } - if incoming.RequestID > 0 { - wc.AddResponseWithID(incoming.RequestID, resp.Raw) - return - } - } - } -} diff --git a/exchanges/wshandler/websocket_connection_types.go b/exchanges/wshandler/websocket_connection_types.go deleted file mode 100644 index 214986a6..00000000 --- a/exchanges/wshandler/websocket_connection_types.go +++ /dev/null @@ -1,25 +0,0 @@ -package wshandler - -import ( - "sync" - "time" - - "github.com/gorilla/websocket" -) - -// WebsocketConnection contains all the data needed to send a message to a WS -type WebsocketConnection struct { - sync.Mutex - Verbose bool - RateLimit float64 - ExchangeName string - URL string - ProxyURL string - Wg sync.WaitGroup - Connection *websocket.Conn - Shutdown chan struct{} - // These are the request IDs and the corresponding response JSON - IDResponses map[int64][]byte - ResponseCheckTimeout time.Duration - ResponseMaxLimit time.Duration -} diff --git a/exchanges/yobit/yobit.go b/exchanges/yobit/yobit.go index 5be8b075..d894fbf8 100644 --- a/exchanges/yobit/yobit.go +++ b/exchanges/yobit/yobit.go @@ -15,7 +15,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/yobit/yobit_wrapper.go b/exchanges/yobit/yobit_wrapper.go index 41ef8599..010804aa 100644 --- a/exchanges/yobit/yobit_wrapper.go +++ b/exchanges/yobit/yobit_wrapper.go @@ -14,7 +14,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/exchanges/zb/zb.go b/exchanges/zb/zb.go index ee95706e..6f15e723 100644 --- a/exchanges/zb/zb.go +++ b/exchanges/zb/zb.go @@ -16,7 +16,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/request" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -86,6 +86,7 @@ func (z *ZB) SetDefaults() { wshandler.WebsocketMessageCorrelationSupported z.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit z.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout + z.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit } // Setup sets user configuration @@ -148,6 +149,13 @@ func (z *ZB) Setup(exch *config.ExchangeConfig) { ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout, ResponseMaxLimit: exch.WebsocketResponseMaxLimit, } + z.Websocket.Orderbook.Setup( + exch.WebsocketOrderbookBufferLimit, + false, + false, + false, + false, + exch.Name) } } diff --git a/exchanges/zb/zb_test.go b/exchanges/zb/zb_test.go index 1d353b27..d38a2959 100644 --- a/exchanges/zb/zb_test.go +++ b/exchanges/zb/zb_test.go @@ -10,7 +10,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/config" "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" ) // Please supply you own test keys here for due diligence testing. diff --git a/exchanges/zb/zb_websocket.go b/exchanges/zb/zb_websocket.go index 083c02ee..825ecac9 100644 --- a/exchanges/zb/zb_websocket.go +++ b/exchanges/zb/zb_websocket.go @@ -12,7 +12,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/currency" exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) @@ -95,7 +95,7 @@ func (z *ZB) WsHandleData() { z.Websocket.DataHandler <- wshandler.TickerData{ Timestamp: time.Unix(0, ticker.Date), Pair: currency.NewPairFromString(cPair[0]), - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: z.GetName(), ClosePrice: ticker.Data.Last, HighPrice: ticker.Data.High, @@ -111,8 +111,8 @@ func (z *ZB) WsHandleData() { } var asks []orderbook.Item - for _, askDepth := range depth.Asks { - ask := askDepth.([]interface{}) + for i := range depth.Asks { + ask := depth.Asks[i].([]interface{}) asks = append(asks, orderbook.Item{ Amount: ask[1].(float64), Price: ask[0].(float64), @@ -120,8 +120,8 @@ func (z *ZB) WsHandleData() { } var bids []orderbook.Item - for _, bidDepth := range depth.Bids { - bid := bidDepth.([]interface{}) + for i := range depth.Bids { + bid := depth.Bids[i].([]interface{}) bids = append(bids, orderbook.Item{ Amount: bid[1].(float64), Price: bid[0].(float64), @@ -133,11 +133,10 @@ func (z *ZB) WsHandleData() { var newOrderBook orderbook.Base newOrderBook.Asks = asks newOrderBook.Bids = bids - newOrderBook.AssetType = "SPOT" + newOrderBook.AssetType = orderbook.Spot newOrderBook.Pair = cPair err = z.Websocket.Orderbook.LoadSnapshot(&newOrderBook, - z.GetName(), true) if err != nil { z.Websocket.DataHandler <- err @@ -146,7 +145,7 @@ func (z *ZB) WsHandleData() { z.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{ Pair: cPair, - Asset: "SPOT", + Asset: orderbook.Spot, Exchange: z.GetName(), } @@ -167,7 +166,7 @@ func (z *ZB) WsHandleData() { z.Websocket.DataHandler <- wshandler.TradeData{ Timestamp: time.Unix(0, t.Date), CurrencyPair: cPair, - AssetType: "SPOT", + AssetType: orderbook.Spot, Exchange: z.GetName(), EventTime: t.Date, Price: t.Price, diff --git a/exchanges/zb/zb_wrapper.go b/exchanges/zb/zb_wrapper.go index ce1e9ac0..cfb6eb38 100644 --- a/exchanges/zb/zb_wrapper.go +++ b/exchanges/zb/zb_wrapper.go @@ -12,7 +12,7 @@ import ( exchange "github.com/thrasher-corp/gocryptotrader/exchanges" "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/go.mod b/go.mod index fefa6171..fb714ed2 100644 --- a/go.mod +++ b/go.mod @@ -6,5 +6,6 @@ require ( github.com/google/go-querystring v1.0.0 github.com/gorilla/mux v1.7.3 github.com/gorilla/websocket v1.4.0 + github.com/toorop/go-pusher v0.0.0-20180521062818-4521e2eb39fb golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f ) diff --git a/go.sum b/go.sum index d839a6b8..cf95f9f8 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/toorop/go-pusher v0.0.0-20180521062818-4521e2eb39fb h1:9kcmLvQdiIecpgVEL3/+J5QIP/ElRBJDljOay0SvqnA= +github.com/toorop/go-pusher v0.0.0-20180521062818-4521e2eb39fb/go.mod h1:VTLqNCX1tXrur6pdIRCl8Q90FR7nw/mEBdyMkWMcsb0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/routines.go b/routines.go index f7d2f1b1..8577f0c5 100644 --- a/routines.go +++ b/routines.go @@ -12,7 +12,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" "github.com/thrasher-corp/gocryptotrader/exchanges/stats" "github.com/thrasher-corp/gocryptotrader/exchanges/ticker" - "github.com/thrasher-corp/gocryptotrader/exchanges/wshandler" + "github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler" log "github.com/thrasher-corp/gocryptotrader/logger" ) diff --git a/testdata/configtest.json b/testdata/configtest.json index 282be76f..62a9f87e 100644 --- a/testdata/configtest.json +++ b/testdata/configtest.json @@ -172,6 +172,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -216,6 +217,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -260,6 +262,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -312,6 +315,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -358,6 +362,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -403,6 +408,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -447,6 +453,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -492,6 +499,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -537,6 +545,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -581,6 +590,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -625,6 +635,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -671,6 +682,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -716,6 +728,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -761,6 +774,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -804,6 +818,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -848,6 +863,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -893,6 +909,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -938,6 +955,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -983,6 +1001,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -1022,12 +1041,13 @@ "name": "LakeBTC", "enabled": true, "verbose": false, - "websocket": false, + "websocket": true, "useSandbox": false, "restPollingDelay": 10, "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -1038,8 +1058,8 @@ "apiUrlSecondary": "NON_DEFAULT_HTTP_LINK_TO_EXCHANGE_API", "proxyAddress": "", "websocketUrl": "NON_DEFAULT_HTTP_LINK_TO_WEBSOCKET_EXCHANGE_API", - "availablePairs": "ETHBTC,USDNGN,USDSGD,EURUSD,USDHKD,BACETH,BTCCHF,BTCGBP,BTCJPY,BTCCAD,BTCEUR,USDCAD,BTCNGN,AUDUSD,GBPUSD,USDJPY,LTCBTC,BCHBTC,USDCHF,NZDUSD,XRPBTC", - "enabledPairs": "ETHBTC", + "availablePairs": "ETHBTC,USDNGN,USDSGD,EURUSD,USDHKD,BACETH,BTCCHF,BTCGBP,BTCJPY,BTCCAD,BTCEUR,USDCAD,BTCNGN,AUDUSD,GBPUSD,USDJPY,LTCBTC,BCHBTC,USDCHF,NZDUSD,XRPBTC,BTCUSD", + "enabledPairs": "BTCUSD,BTCEUR,LTCBTC", "baseCurrencies": "USD,EUR,HKD,AUD,GBP,NZD,JPY,SGD,NGN,CHF,CAD", "assetTypes": "SPOT", "supportsAutoPairUpdates": true, @@ -1071,6 +1091,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -1114,6 +1135,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -1159,6 +1181,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -1204,6 +1227,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -1249,6 +1273,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -1296,6 +1321,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, @@ -1341,6 +1367,7 @@ "httpTimeout": 15000000000, "websocketResponseCheckTimeout": 30000000, "websocketResponseMaxLimit": 7000000000, + "websocketOrderbookBufferLimit": 5, "httpUserAgent": "", "httpDebugging": false, "authenticatedApiSupport": false, diff --git a/tools/exchange_template/exchange_template.go b/tools/exchange_template/exchange_template.go index f671f510..2276365c 100644 --- a/tools/exchange_template/exchange_template.go +++ b/tools/exchange_template/exchange_template.go @@ -10,6 +10,7 @@ import ( "github.com/thrasher-corp/gocryptotrader/common" "github.com/thrasher-corp/gocryptotrader/config" + "github.com/thrasher-corp/gocryptotrader/exchanges/orderbook" ) const ( @@ -120,7 +121,7 @@ func main() { newExchConfig.RESTPollingDelay = 10 newExchConfig.APIKey = "Key" newExchConfig.APISecret = "Secret" - newExchConfig.AssetTypes = "SPOT" + newExchConfig.AssetTypes = orderbook.Spot configTestFile.Exchanges = append(configTestFile.Exchanges, newExchConfig) // TODO sorting function so exchanges are in alphabetical order - low priority diff --git a/tools/exchange_template/main_file.tmpl b/tools/exchange_template/main_file.tmpl index a19b1cea..b72be2b7 100644 --- a/tools/exchange_template/main_file.tmpl +++ b/tools/exchange_template/main_file.tmpl @@ -47,7 +47,7 @@ func ({{.Variable}} *{{.CapitalName}}) SetDefaults() { common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout)) {{.Variable}}.APIUrlDefault = {{.Name}}APIURL {{.Variable}}.APIUrl = {{.Variable}}.APIUrlDefault - {{.Variable}}.Websocket = wshandler.New() + {{.Variable}}.Websocket = monitor.New() {{.Variable}}.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit {{.Variable}}.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout } diff --git a/tools/exchange_template/wrapper_file.tmpl b/tools/exchange_template/wrapper_file.tmpl index 2217c869..2e169090 100644 --- a/tools/exchange_template/wrapper_file.tmpl +++ b/tools/exchange_template/wrapper_file.tmpl @@ -187,20 +187,20 @@ func ({{.Variable}} *{{.CapitalName}}) GetFeeByType(feeBuilder *exchange.FeeBuil // SubscribeToWebsocketChannels appends to ChannelsToSubscribe // which lets websocket.manageSubscriptions handle subscribing -func ({{.Variable}} *{{.CapitalName}}) SubscribeToWebsocketChannels(channels []wshandler.WebsocketChannelSubscription) error { +func ({{.Variable}} *{{.CapitalName}}) SubscribeToWebsocketChannels(channels []monitor.WebsocketChannelSubscription) error { {{.Variable}}.Websocket.SubscribeToChannels(channels) return nil } // UnsubscribeToWebsocketChannels removes from ChannelsToSubscribe // which lets websocket.manageSubscriptions handle unsubscribing -func ({{.Variable}} *{{.CapitalName}}) UnsubscribeToWebsocketChannels(channels []wshandler.WebsocketChannelSubscription) error { +func ({{.Variable}} *{{.CapitalName}}) UnsubscribeToWebsocketChannels(channels []monitor.WebsocketChannelSubscription) error { {{.Variable}}.Websocket.UnubscribeToChannels(channels) return nil } // GetSubscriptions returns a copied list of subscriptions -func ({{.Variable}} *{{.CapitalName}}) GetSubscriptions() ([]wshandler.WebsocketChannelSubscription, error) { +func ({{.Variable}} *{{.CapitalName}}) GetSubscriptions() ([]monitor.WebsocketChannelSubscription, error) { return nil, common.ErrNotYetImplemented }