d0zingcat/gocryptotrader
1
0
Fork 0
mirror of https://github.com/d0zingcat/gocryptotrader.git synced 2026-05-30 23:16:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Websocket orderbook buffering (#333)

Browse Source 
* Initial commit setting up a map orderbook system with a buffer. It will write to the buffer, sort apply to main orderbook and then process.

* Moves namespaces again

* Updates orderbook to use a sweet new WebsocketOrderbookUpdate type to handle all updates whether its using ID or not. So good. Adds many tests

* Starting to implement orderbook update handling per exchange. Updates namespaces again. Hopefuylly will find a way to update via ID not timestamp, too many endpoints dont provide update timestamps

* Changes orderbookbuffer to use BufferUpdate type instead of orderbook.Base to achieve more functionality and no need for type conversion functions. Updates tests

* Updates all instances of ws.orderbook.Update. Simplifies some orderbook logic

* Introduces toggleable buffer. Renames orderbooks. Completes implementation for everywhere but OKGroup due to hash calculation

* Implements orderbook update for okgroup, but forgets about the orderbook hash checking

* Fixes okgroup checksum calculation. Fixes linting issue. Removes redundant Kraken tests.

* Introduces sorting toggle and separates from buffer toggle. Uses benchmarks to highlight performance gains

* Fixes Gemini rate limit and parsing. Removes comments and fixes typos

* Fixes bitfinex orderbook processing

* Inbuilt sorting, minor fixes for websocket implementations. Improves test coverage

* Adds surprise LakeBTC websocket support

* Fixes data race

* Fixes rebasing issues due to namespace movements

* Addresses PR nits: moves folder namespace from ws to websocket. Removes line spaces in imports. Fixes lakebtc websocket returns and defer fucntions. Fixes comments

* Adds poloniex orderook sorting support

* Enables bitstamp and hitbtc orderbook sorting. Fixes poloniex's sorting

* Renames namespaces and combines monitor and connection into wshandler. Removes unused SPOT const. Changes how orderbook stuff is loaded. It is done in startup with a setup. Removes exchange name from loadsnapshot as well

* Removes the connection.go from rebasing issues. Removes error response from functions used in goroutines

* Fixes test with exchange name output change

* Fixes issues where copy and paste and replace all were used poorly
This commit is contained in:
Scott
2019-08-13 09:32:59 +10:00
committed by Adrian Gallagher
parent 2078ba907f
commit 0fbf8b172a
110 changed files with 2197 additions and 1909 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
exchanges/binance/binance.go
View File

@@ -16,7 +16,7 @@ import (
exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
"github.com/thrasher-corp/gocryptotrader/exchanges/request"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
log "github.com/thrasher-corp/gocryptotrader/logger"
)
@@ -101,6 +101,7 @@ func (b *Binance) SetDefaults() {
wshandler.WebsocketOrderbookSupported
b.WebsocketResponseMaxLimit = exchange.DefaultWebsocketResponseMaxLimit
b.WebsocketResponseCheckTimeout = exchange.DefaultWebsocketResponseCheckTimeout
b.WebsocketOrderbookBufferLimit = exchange.DefaultWebsocketOrderbookBufferLimit
}
// Setup takes in the supplied exchange configuration details and sets params
@@ -152,7 +153,6 @@ func (b *Binance) Setup(exch *config.ExchangeConfig) {
if err != nil {
log.Fatal(err)
}
b.WebsocketConn = &wshandler.WebsocketConnection{
ExchangeName: b.Name,
URL: b.Websocket.GetWebsocketURL(),
@@ -161,6 +161,13 @@ func (b *Binance) Setup(exch *config.ExchangeConfig) {
ResponseCheckTimeout: exch.WebsocketResponseCheckTimeout,
ResponseMaxLimit: exch.WebsocketResponseMaxLimit,
}
b.Websocket.Orderbook.Setup(
exch.WebsocketOrderbookBufferLimit,
true,
true,
true,
false,
exch.Name)
}
}

177
exchanges/binance/binance_websocket.go
View File

@@ -6,7 +6,6 @@ import (
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/gorilla/websocket"
@@ -15,112 +14,14 @@ import (
exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wsorderbook"
)
const (
binanceDefaultWebsocketURL = "wss://stream.binance.com:9443"
)
var lastUpdateID map[string]int64
var m sync.Mutex
// SeedLocalCache seeds depth data
func (b *Binance) SeedLocalCache(p currency.Pair) error {
var newOrderBook orderbook.Base
formattedPair := exchange.FormatExchangeCurrency(b.Name, p)
orderbookNew, err := b.GetOrderBook(
OrderBookDataRequestParams{
Symbol: formattedPair.String(),
Limit: 1000,
})
if err != nil {
return err
}
m.Lock()
if lastUpdateID == nil {
lastUpdateID = make(map[string]int64)
}
lastUpdateID[formattedPair.String()] = orderbookNew.LastUpdateID
m.Unlock()
for _, bids := range orderbookNew.Bids {
newOrderBook.Bids = append(newOrderBook.Bids,
orderbook.Item{Amount: bids.Quantity, Price: bids.Price})
}
for _, Asks := range orderbookNew.Asks {
newOrderBook.Asks = append(newOrderBook.Asks,
orderbook.Item{Amount: Asks.Quantity, Price: Asks.Price})
}
newOrderBook.Pair = currency.NewPairFromString(formattedPair.String())
newOrderBook.AssetType = ticker.Spot
return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, b.GetName(), false)
}
// UpdateLocalCache updates and returns the most recent iteration of the orderbook
func (b *Binance) UpdateLocalCache(ob *WebsocketDepthStream) error {
m.Lock()
ID, ok := lastUpdateID[ob.Pair]
if !ok {
m.Unlock()
return fmt.Errorf("%v - Unable to find lastUpdateID", b.Name)
}
if ob.LastUpdateID+1 <= ID || ID >= ob.LastUpdateID+1 {
// Drop update, out of order
m.Unlock()
return nil
}
lastUpdateID[ob.Pair] = ob.LastUpdateID
m.Unlock()
var updateBid, updateAsk []orderbook.Item
for _, bidsToUpdate := range ob.UpdateBids {
var priceToBeUpdated orderbook.Item
for i, bids := range bidsToUpdate.([]interface{}) {
switch i {
case 0:
priceToBeUpdated.Price, _ = strconv.ParseFloat(bids.(string), 64)
case 1:
priceToBeUpdated.Amount, _ = strconv.ParseFloat(bids.(string), 64)
}
}
updateBid = append(updateBid, priceToBeUpdated)
}
for _, asksToUpdate := range ob.UpdateAsks {
var priceToBeUpdated orderbook.Item
for i, asks := range asksToUpdate.([]interface{}) {
switch i {
case 0:
priceToBeUpdated.Price, _ = strconv.ParseFloat(asks.(string), 64)
case 1:
priceToBeUpdated.Amount, _ = strconv.ParseFloat(asks.(string), 64)
}
}
updateAsk = append(updateAsk, priceToBeUpdated)
}
updatedTime := time.Unix(ob.Timestamp, 0)
currencyPair := currency.NewPairFromString(ob.Pair)
return b.Websocket.Orderbook.Update(updateBid,
updateAsk,
currencyPair,
updatedTime,
b.GetName(),
"SPOT")
}
// WSConnect intiates a websocket connection
func (b *Binance) WSConnect() error {
if !b.Websocket.IsEnabled() || !b.IsEnabled() {
@@ -175,11 +76,9 @@ func (b *Binance) WSConnect() error {
// WsHandleData handles websocket data from WsReadData
func (b *Binance) WsHandleData() {
b.Websocket.Wg.Add(1)
defer func() {
b.Websocket.Wg.Done()
}()
for {
select {
case <-b.Websocket.ShutdownC:
@@ -234,7 +133,7 @@ func (b *Binance) WsHandleData() {
Price: price,
Amount: amount,
Exchange: b.GetName(),
AssetType: "SPOT",
AssetType: orderbook.Spot,
Side: trade.EventType,
}
continue
@@ -309,7 +208,7 @@ func (b *Binance) WsHandleData() {
currencyPair := currency.NewPairFromString(depth.Pair)
b.Websocket.DataHandler <- wshandler.WebsocketOrderbookUpdate{
Pair: currencyPair,
Asset: "SPOT",
Asset: orderbook.Spot,
Exchange: b.GetName(),
}
continue
@@ -317,3 +216,71 @@ func (b *Binance) WsHandleData() {
}
}
}
// SeedLocalCache seeds depth data
func (b *Binance) SeedLocalCache(p currency.Pair) error {
var newOrderBook orderbook.Base
formattedPair := exchange.FormatExchangeCurrency(b.Name, p)
orderbookNew, err := b.GetOrderBook(
OrderBookDataRequestParams{
Symbol: formattedPair.String(),
Limit: 1000,
})
if err != nil {
return err
}
for i := range orderbookNew.Bids {
newOrderBook.Bids = append(newOrderBook.Bids,
orderbook.Item{Amount: orderbookNew.Bids[i].Quantity, Price: orderbookNew.Bids[i].Price})
}
for i := range orderbookNew.Asks {
newOrderBook.Asks = append(newOrderBook.Asks,
orderbook.Item{Amount: orderbookNew.Asks[i].Quantity, Price: orderbookNew.Asks[i].Price})
}
newOrderBook.LastUpdated = time.Unix(orderbookNew.LastUpdateID, 0)
newOrderBook.Pair = currency.NewPairFromString(formattedPair.String())
newOrderBook.AssetType = ticker.Spot
return b.Websocket.Orderbook.LoadSnapshot(&newOrderBook, false)
}
// UpdateLocalCache updates and returns the most recent iteration of the orderbook
func (b *Binance) UpdateLocalCache(wsdp *WebsocketDepthStream) error {
var updateBid, updateAsk []orderbook.Item
for i := range wsdp.UpdateBids {
var priceToBeUpdated orderbook.Item
for i, bids := range wsdp.UpdateBids[i].([]interface{}) {
switch i {
case 0:
priceToBeUpdated.Price, _ = strconv.ParseFloat(bids.(string), 64)
case 1:
priceToBeUpdated.Amount, _ = strconv.ParseFloat(bids.(string), 64)
}
}
updateBid = append(updateBid, priceToBeUpdated)
}
for i := range wsdp.UpdateAsks {
var priceToBeUpdated orderbook.Item
for i, asks := range wsdp.UpdateAsks[i].([]interface{}) {
switch i {
case 0:
priceToBeUpdated.Price, _ = strconv.ParseFloat(asks.(string), 64)
case 1:
priceToBeUpdated.Amount, _ = strconv.ParseFloat(asks.(string), 64)
}
}
updateAsk = append(updateAsk, priceToBeUpdated)
}
currencyPair := currency.NewPairFromString(wsdp.Pair)
return b.Websocket.Orderbook.Update(&wsorderbook.WebsocketOrderbookUpdate{
Bids: updateBid,
Asks: updateAsk,
CurrencyPair: currencyPair,
UpdateID: wsdp.LastUpdateID,
AssetType: orderbook.Spot,
})
}

2
exchanges/binance/binance_wrapper.go
View File

@@ -13,7 +13,7 @@ import (
exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
"github.com/thrasher-corp/gocryptotrader/exchanges/orderbook"
"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
"github.com/thrasher-corp/gocryptotrader/exchanges/wshandler"
"github.com/thrasher-corp/gocryptotrader/exchanges/websocket/wshandler"
log "github.com/thrasher-corp/gocryptotrader/logger"
)